Restores downloading of metrics on test failure

DistTestCore/DistTest.cs

@@ -1,4 +1,5 @@
 using DistTestCore.CodexLogsAndMetrics;
+using DistTestCore.Metrics;
 using NUnit.Framework;
 
 namespace DistTestCore
@@ -76,7 +77,7 @@ namespace DistTestCore
                 {
                     Log("Downloading all CodexNode logs and metrics because of test failure...");
                     DownloadAllLogs();
-                    //k8sManager.DownloadAllMetrics();
+                    DownloadAllMetrics();
                 }
                 else
                 {
@@ -102,10 +103,29 @@ namespace DistTestCore
 
         private void DownloadAllLogs()
         {
-            var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
+            OnEachCodexNode(node =>
-            foreach (var node in allNodes) 
             {
                 lifecycle.DownloadLog(node);
+            });
+        }
+
+        private void DownloadAllMetrics()
+        {
+            var metricsDownloader = new MetricsDownloader(lifecycle.Log);
+
+            OnEachCodexNode(node =>
+            {
+                var m = (MetricsAccess)node.Metrics;
+                metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m);
+            });
+        }
+
+        private void OnEachCodexNode(Action<OnlineCodexNode> action)
+        {
+            var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
+            foreach (var node in allNodes)
+            {
+                action(node);
             }
         }
 

DistTestCore/Metrics/MetricsAccess.cs

@@ -37,6 +37,11 @@ namespace DistTestCore.Metrics
             Assert.That(metricValue, constraint, message);
         }
 
+        public Metrics? GetAllMetrics()
+        {
+            return query.GetAllMetricsForNode(node);
+        }
+
         private MetricsSet GetMetricWithTimeout(string metricName)
         {
             var start = DateTime.UtcNow;

DistTestCore/Metrics/MetricsDownloader.cs

@@ -6,39 +6,21 @@ namespace DistTestCore.Metrics
     public class MetricsDownloader
     {
         private readonly TestLog log;
-        private readonly Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses;
 
-        public MetricsDownloader(TestLog log, Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses)
+        public MetricsDownloader(TestLog log)
         {
             this.log = log;
-            this.activePrometheuses = activePrometheuses;
         }
 
-        public void DownloadAllMetrics()
+        public void DownloadAllMetricsForNode(string nodeName, MetricsAccess access)
         {
-            foreach (var pair in activePrometheuses)
+            var metrics = access.GetAllMetrics();
-            {
-                DownloadAllMetrics(pair.Key, pair.Value);
-            }
-        }
-
-        private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes)
-        {
-            foreach (var node in nodes)
-            {
-                DownloadAllMetricsForNode(query, node);
-            }
-        }
-
-        private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node)
-        {
-            var metrics = query.GetAllMetricsForNode(node.CodexAccess.Container);
             if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return;
 
             var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray();
             var map = CreateValueMap(metrics);
 
-            WriteToFile(node.GetName(), headers, map);
+            WriteToFile(nodeName, headers, map);
         }
 
         private void WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)