Extracts query logic from metricsAccess object

CodexDistTestCore/MetricsAccess.cs

@@ -1,5 +1,4 @@
-using CodexDistTestCore.Config;
+using NUnit.Framework;
-using NUnit.Framework;
 using NUnit.Framework.Constraints;
 
 namespace CodexDistTestCore
@@ -11,33 +10,34 @@ namespace CodexDistTestCore
 
     public class MetricsAccess : IMetricsAccess
     {
-        private readonly K8sCluster k8sCluster = new K8sCluster();
+        private readonly MetricsQuery query;
-        private readonly Http http;
         private readonly OnlineCodexNode[] nodes;
 
-        public MetricsAccess(PrometheusInfo prometheusInfo, OnlineCodexNode[] nodes)
+        public MetricsAccess(MetricsQuery query, OnlineCodexNode[] nodes)
         {
-            http = new Http(
+            this.query = query;
-                k8sCluster.GetIp(),
-                prometheusInfo.ServicePort,
-                "api/v1");
             this.nodes = nodes;
         }
 
         public void AssertThat(IOnlineCodexNode node, string metricName, IResolveConstraint constraint, string message = "")
         {
-            var metricValue = GetMetricWithTimeout(metricName, node);
+            var n = (OnlineCodexNode)node;
+            CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " +
+                "(This CodexNode is tracked by a different instance.)");
+
+            var metricSet = GetMetricWithTimeout(metricName, n);
+            var metricValue = metricSet.Values[0].Value;
             Assert.That(metricValue, constraint, message);
         }
 
-        private double GetMetricWithTimeout(string metricName, IOnlineCodexNode node)
+        private MetricsSet GetMetricWithTimeout(string metricName, OnlineCodexNode node)
         {
             var start = DateTime.UtcNow;
 
             while (true)
             {
                 var mostRecent = GetMostRecent(metricName, node);
-                if (mostRecent != null) return Convert.ToDouble(mostRecent);
+                if (mostRecent != null) return mostRecent;
                 if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
                 {
                     Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
@@ -48,60 +48,14 @@ namespace CodexDistTestCore
             }
         }
 
-        private object? GetMostRecent(string metricName, IOnlineCodexNode node)
+        private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node)
         {
-            var n = (OnlineCodexNode)node;
+            var result = query.GetMostRecent(metricName);
-            CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " +
+            if (result == null) return null;
-                "(This CodexNode is tracked by a different instance.)");
 
-            var response = GetMetric(metricName);
-            if (response == null) return null;
-
-            var value = GetValueFromResponse(n, response);
-            if (value == null) return null;
-            if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string].");
-            return value[1];
-        }
-
-        private PrometheusQueryResponse? GetMetric(string metricName)
-        {
-            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}[12h])");
-            if (response.status != "success") return null;
-            return response;
-        }
-
-        private object[]? GetValueFromResponse(OnlineCodexNode node, PrometheusQueryResponse response)
-        {
             var pod = node.Group.PodInfo!;
-            var forNode = response.data.result.SingleOrDefault(d => d.metric.instance == $"{pod.Ip}:{node.Container.MetricsPort}");
+            var instance = $"{pod.Ip}:{node.Container.MetricsPort}";
-            if (forNode == null) return null;
+            return result.Sets.SingleOrDefault(r => r.Instance == instance);
-            if (forNode.value == null || forNode.value.Length == 0) return null;
-            return forNode.value;
         }
     }
-
-    public class PrometheusQueryResponse
-    {
-        public string status { get; set; } = string.Empty;
-        public PrometheusQueryResponseData data { get; set; } = new();
-    }
-
-    public class PrometheusQueryResponseData
-    {
-        public string resultType { get; set; } = string.Empty;
-        public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty<PrometheusQueryResponseDataResultEntry>();
-    }
-
-    public class PrometheusQueryResponseDataResultEntry
-    {
-        public ResultEntryMetric metric { get; set; } = new();
-        public object[] value { get; set; } = Array.Empty<object>();
-    }
-
-    public class ResultEntryMetric
-    {
-        public string __name__ { get; set; } = string.Empty;
-        public string instance { get; set; } = string.Empty;
-        public string job { get; set; } = string.Empty;
-    }
 }

CodexDistTestCore/MetricsAggregator.cs

@@ -8,7 +8,7 @@ namespace CodexDistTestCore
         private readonly NumberSource prometheusNumberSource = new NumberSource(0);
         private readonly TestLog log;
         private readonly K8sManager k8sManager;
-        private readonly Dictionary<PrometheusInfo, OnlineCodexNode[]> activePrometheuses = new Dictionary<PrometheusInfo, OnlineCodexNode[]>();
+        private readonly Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses = new Dictionary<MetricsQuery, OnlineCodexNode[]>();
 
         public MetricsAggregator(TestLog log, K8sManager k8sManager)
         {
@@ -29,10 +29,11 @@ namespace CodexDistTestCore
 
             var config = GeneratePrometheusConfig(nodes);
             var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber());
-            activePrometheuses.Add(prometheus, nodes);
+            var query = new MetricsQuery(prometheus);
+            activePrometheuses.Add(query, nodes);
 
             log.Log("Metrics service started.");
-            return new MetricsAccess(prometheus, nodes);
+            return new MetricsAccess(query, nodes);
         }
 
         public void DownloadAllMetrics()
@@ -51,7 +52,6 @@ namespace CodexDistTestCore
             config += "    metrics_path: /metrics\n";
             config += "    static_configs:\n";
             config += "      - targets:\n";
-            config += "          - 'prometheus:9090'\n";
 
             foreach (var node in nodes)
             {

CodexDistTestCore/MetricsQuery.cs

@@ -0,0 +1,160 @@
+using CodexDistTestCore.Config;
+using System.Globalization;
+
+namespace CodexDistTestCore
+{
+    public class MetricsQuery
+    {
+        private readonly K8sCluster k8sCluster = new K8sCluster();
+        private readonly Http http;
+
+        public MetricsQuery(PrometheusInfo prometheusInfo)
+        {
+            http = new Http(
+                k8sCluster.GetIp(),
+                prometheusInfo.ServicePort,
+                "api/v1");
+        }
+
+        public Metrics? GetMostRecent(string metricName)
+        {
+            var response = GetLastOverTime(metricName);
+            if (response == null) return null;
+
+            return new Metrics
+            {
+                Sets = response.data.result.Select(r =>
+                {
+                    return new MetricsSet
+                    {
+                        Instance = r.metric.instance,
+                        Values = MapSingleValue(r.value)
+                    };
+                }).ToArray()
+            };
+        }
+
+        public Metrics? GetMetrics(string metricName)
+        {
+            var response = GetAll(metricName);
+            if (response == null) return null;
+
+            return new Metrics
+            {
+                Sets = response.data.result.Select(r =>
+                {
+                    return new MetricsSet
+                    {
+                        Instance = r.metric.instance,
+                        Values = MapMultipleValues(r.values)
+                    };
+                }).ToArray()
+            };
+        }
+
+        private PrometheusQueryResponse? GetLastOverTime(string metricName)
+        {
+            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{GetQueryTimeRange()})");
+            if (response.status != "success") return null;
+            return response;
+        }
+
+        private PrometheusQueryResponse? GetAll(string metricName)
+        {
+            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}");
+            if (response.status != "success") return null;
+            return response;
+        }
+
+        private MetricsSetValue[] MapSingleValue(object[] value)
+        {
+            if (value != null && value.Length > 0)
+            {
+                return new[]
+                {
+                    MapValue(value)
+                };
+            }
+            return Array.Empty<MetricsSetValue>();
+        }
+
+        private MetricsSetValue[] MapMultipleValues(object[][] values)
+        {
+            if (values != null && values.Length > 0)
+            {
+                return values.Select(v => MapValue(v)).ToArray();
+            }
+            return Array.Empty<MetricsSetValue>();
+        }
+
+        private MetricsSetValue MapValue(object[] value)
+        {
+            if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string].");
+
+            return new MetricsSetValue
+            {
+                Timestamp = ToTimestamp(value[0]),
+                Value = ToValue(value[1])
+            };            
+        }
+
+        private string GetQueryTimeRange()
+        {
+            return "[12h]";
+        }
+
+        private double ToValue(object v)
+        {
+            return Convert.ToDouble(v, CultureInfo.InvariantCulture);
+        }
+
+        private DateTime ToTimestamp(object v)
+        {
+            var unixSeconds = ToValue(v);
+            return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
+        }
+    }
+
+    public class Metrics
+    {
+        public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>();
+    }
+
+    public class MetricsSet
+    {
+        public string Instance { get; set; } = string.Empty;
+        public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
+    }
+
+    public class MetricsSetValue
+    {
+        public DateTime Timestamp { get; set; }
+        public double Value { get; set; }
+    }
+
+    public class PrometheusQueryResponse
+    {
+        public string status { get; set; } = string.Empty;
+        public PrometheusQueryResponseData data { get; set; } = new();
+    }
+
+    public class PrometheusQueryResponseData
+    {
+        public string resultType { get; set; } = string.Empty;
+        public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty<PrometheusQueryResponseDataResultEntry>();
+    }
+
+    public class PrometheusQueryResponseDataResultEntry
+    {
+        public ResultEntryMetric metric { get; set; } = new();
+        public object[] value { get; set; } = Array.Empty<object>();
+        public object[][] values { get; set; } = Array.Empty<object[]>();
+    }
+
+    public class ResultEntryMetric
+    {
+        public string __name__ { get; set; } = string.Empty;
+        public string instance { get; set; } = string.Empty;
+        public string job { get; set; } = string.Empty;
+    }
+}