From 935c2320a783240682e9248b4903e637fbf59c7d Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 29 Mar 2023 11:29:43 +0200 Subject: [PATCH] Extracts query logic from metricsAccess object --- CodexDistTestCore/MetricsAccess.cs | 80 +++---------- CodexDistTestCore/MetricsAggregator.cs | 8 +- CodexDistTestCore/MetricsQuery.cs | 160 +++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 67 deletions(-) create mode 100644 CodexDistTestCore/MetricsQuery.cs diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/MetricsAccess.cs index fc308b9..534d478 100644 --- a/CodexDistTestCore/MetricsAccess.cs +++ b/CodexDistTestCore/MetricsAccess.cs @@ -1,5 +1,4 @@ -using CodexDistTestCore.Config; -using NUnit.Framework; +using NUnit.Framework; using NUnit.Framework.Constraints; namespace CodexDistTestCore @@ -11,33 +10,34 @@ namespace CodexDistTestCore public class MetricsAccess : IMetricsAccess { - private readonly K8sCluster k8sCluster = new K8sCluster(); - private readonly Http http; + private readonly MetricsQuery query; private readonly OnlineCodexNode[] nodes; - public MetricsAccess(PrometheusInfo prometheusInfo, OnlineCodexNode[] nodes) + public MetricsAccess(MetricsQuery query, OnlineCodexNode[] nodes) { - http = new Http( - k8sCluster.GetIp(), - prometheusInfo.ServicePort, - "api/v1"); + this.query = query; this.nodes = nodes; } public void AssertThat(IOnlineCodexNode node, string metricName, IResolveConstraint constraint, string message = "") { - var metricValue = GetMetricWithTimeout(metricName, node); + var n = (OnlineCodexNode)node; + CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " + + "(This CodexNode is tracked by a different instance.)"); + + var metricSet = GetMetricWithTimeout(metricName, n); + var metricValue = metricSet.Values[0].Value; Assert.That(metricValue, constraint, message); } - private double GetMetricWithTimeout(string metricName, IOnlineCodexNode node) + private MetricsSet GetMetricWithTimeout(string metricName, OnlineCodexNode node) { var start = DateTime.UtcNow; while (true) { var mostRecent = GetMostRecent(metricName, node); - if (mostRecent != null) return Convert.ToDouble(mostRecent); + if (mostRecent != null) return mostRecent; if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout()) { Assert.Fail($"Timeout: Unable to get metric '{metricName}'."); @@ -48,60 +48,14 @@ namespace CodexDistTestCore } } - private object? GetMostRecent(string metricName, IOnlineCodexNode node) + private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node) { - var n = (OnlineCodexNode)node; - CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " + - "(This CodexNode is tracked by a different instance.)"); + var result = query.GetMostRecent(metricName); + if (result == null) return null; - var response = GetMetric(metricName); - if (response == null) return null; - - var value = GetValueFromResponse(n, response); - if (value == null) return null; - if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); - return value[1]; - } - - private PrometheusQueryResponse? GetMetric(string metricName) - { - var response = http.HttpGetJson($"query?query=last_over_time({metricName}[12h])"); - if (response.status != "success") return null; - return response; - } - - private object[]? GetValueFromResponse(OnlineCodexNode node, PrometheusQueryResponse response) - { var pod = node.Group.PodInfo!; - var forNode = response.data.result.SingleOrDefault(d => d.metric.instance == $"{pod.Ip}:{node.Container.MetricsPort}"); - if (forNode == null) return null; - if (forNode.value == null || forNode.value.Length == 0) return null; - return forNode.value; + var instance = $"{pod.Ip}:{node.Container.MetricsPort}"; + return result.Sets.SingleOrDefault(r => r.Instance == instance); } } - - public class PrometheusQueryResponse - { - public string status { get; set; } = string.Empty; - public PrometheusQueryResponseData data { get; set; } = new(); - } - - public class PrometheusQueryResponseData - { - public string resultType { get; set; } = string.Empty; - public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); - } - - public class PrometheusQueryResponseDataResultEntry - { - public ResultEntryMetric metric { get; set; } = new(); - public object[] value { get; set; } = Array.Empty(); - } - - public class ResultEntryMetric - { - public string __name__ { get; set; } = string.Empty; - public string instance { get; set; } = string.Empty; - public string job { get; set; } = string.Empty; - } } diff --git a/CodexDistTestCore/MetricsAggregator.cs b/CodexDistTestCore/MetricsAggregator.cs index 2e4696b..2fc7b27 100644 --- a/CodexDistTestCore/MetricsAggregator.cs +++ b/CodexDistTestCore/MetricsAggregator.cs @@ -8,7 +8,7 @@ namespace CodexDistTestCore private readonly NumberSource prometheusNumberSource = new NumberSource(0); private readonly TestLog log; private readonly K8sManager k8sManager; - private readonly Dictionary activePrometheuses = new Dictionary(); + private readonly Dictionary activePrometheuses = new Dictionary(); public MetricsAggregator(TestLog log, K8sManager k8sManager) { @@ -29,10 +29,11 @@ namespace CodexDistTestCore var config = GeneratePrometheusConfig(nodes); var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber()); - activePrometheuses.Add(prometheus, nodes); + var query = new MetricsQuery(prometheus); + activePrometheuses.Add(query, nodes); log.Log("Metrics service started."); - return new MetricsAccess(prometheus, nodes); + return new MetricsAccess(query, nodes); } public void DownloadAllMetrics() @@ -51,7 +52,6 @@ namespace CodexDistTestCore config += " metrics_path: /metrics\n"; config += " static_configs:\n"; config += " - targets:\n"; - config += " - 'prometheus:9090'\n"; foreach (var node in nodes) { diff --git a/CodexDistTestCore/MetricsQuery.cs b/CodexDistTestCore/MetricsQuery.cs new file mode 100644 index 0000000..750aa52 --- /dev/null +++ b/CodexDistTestCore/MetricsQuery.cs @@ -0,0 +1,160 @@ +using CodexDistTestCore.Config; +using System.Globalization; + +namespace CodexDistTestCore +{ + public class MetricsQuery + { + private readonly K8sCluster k8sCluster = new K8sCluster(); + private readonly Http http; + + public MetricsQuery(PrometheusInfo prometheusInfo) + { + http = new Http( + k8sCluster.GetIp(), + prometheusInfo.ServicePort, + "api/v1"); + } + + public Metrics? GetMostRecent(string metricName) + { + var response = GetLastOverTime(metricName); + if (response == null) return null; + + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Instance = r.metric.instance, + Values = MapSingleValue(r.value) + }; + }).ToArray() + }; + } + + public Metrics? GetMetrics(string metricName) + { + var response = GetAll(metricName); + if (response == null) return null; + + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Instance = r.metric.instance, + Values = MapMultipleValues(r.values) + }; + }).ToArray() + }; + } + + private PrometheusQueryResponse? GetLastOverTime(string metricName) + { + var response = http.HttpGetJson($"query?query=last_over_time({metricName}{GetQueryTimeRange()})"); + if (response.status != "success") return null; + return response; + } + + private PrometheusQueryResponse? GetAll(string metricName) + { + var response = http.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return response; + } + + private MetricsSetValue[] MapSingleValue(object[] value) + { + if (value != null && value.Length > 0) + { + return new[] + { + MapValue(value) + }; + } + return Array.Empty(); + } + + private MetricsSetValue[] MapMultipleValues(object[][] values) + { + if (values != null && values.Length > 0) + { + return values.Select(v => MapValue(v)).ToArray(); + } + return Array.Empty(); + } + + private MetricsSetValue MapValue(object[] value) + { + if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); + + return new MetricsSetValue + { + Timestamp = ToTimestamp(value[0]), + Value = ToValue(value[1]) + }; + } + + private string GetQueryTimeRange() + { + return "[12h]"; + } + + private double ToValue(object v) + { + return Convert.ToDouble(v, CultureInfo.InvariantCulture); + } + + private DateTime ToTimestamp(object v) + { + var unixSeconds = ToValue(v); + return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds); + } + } + + public class Metrics + { + public MetricsSet[] Sets { get; set; } = Array.Empty(); + } + + public class MetricsSet + { + public string Instance { get; set; } = string.Empty; + public MetricsSetValue[] Values { get; set; } = Array.Empty(); + } + + public class MetricsSetValue + { + public DateTime Timestamp { get; set; } + public double Value { get; set; } + } + + public class PrometheusQueryResponse + { + public string status { get; set; } = string.Empty; + public PrometheusQueryResponseData data { get; set; } = new(); + } + + public class PrometheusQueryResponseData + { + public string resultType { get; set; } = string.Empty; + public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); + } + + public class PrometheusQueryResponseDataResultEntry + { + public ResultEntryMetric metric { get; set; } = new(); + public object[] value { get; set; } = Array.Empty(); + public object[][] values { get; set; } = Array.Empty(); + } + + public class ResultEntryMetric + { + public string __name__ { get; set; } = string.Empty; + public string instance { get; set; } = string.Empty; + public string job { get; set; } = string.Empty; + } +}