From f70ce8e8bb00d24a0db379eeacc59bb4aeae6b4d Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 30 Mar 2023 10:43:17 +0200 Subject: [PATCH] implements downloading of metrics on test failure --- CodexDistTestCore/MetricsAccess.cs | 7 +- CodexDistTestCore/MetricsAggregator.cs | 2 + CodexDistTestCore/MetricsDownloader.cs | 97 ++++++++++++++++++++++++++ CodexDistTestCore/MetricsQuery.cs | 62 +++++++++++----- CodexDistTestCore/TestLog.cs | 11 +-- Tests/BasicTests/SimpleTests.cs | 24 ++++--- 6 files changed, 166 insertions(+), 37 deletions(-) create mode 100644 CodexDistTestCore/MetricsDownloader.cs diff --git a/CodexDistTestCore/MetricsAccess.cs b/CodexDistTestCore/MetricsAccess.cs index 534d478..dbda2a8 100644 --- a/CodexDistTestCore/MetricsAccess.cs +++ b/CodexDistTestCore/MetricsAccess.cs @@ -50,12 +50,9 @@ namespace CodexDistTestCore private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node) { - var result = query.GetMostRecent(metricName); + var result = query.GetMostRecent(metricName, node); if (result == null) return null; - - var pod = node.Group.PodInfo!; - var instance = $"{pod.Ip}:{node.Container.MetricsPort}"; - return result.Sets.SingleOrDefault(r => r.Instance == instance); + return result.Sets.LastOrDefault(); } } } diff --git a/CodexDistTestCore/MetricsAggregator.cs b/CodexDistTestCore/MetricsAggregator.cs index 2fc7b27..7b08f3b 100644 --- a/CodexDistTestCore/MetricsAggregator.cs +++ b/CodexDistTestCore/MetricsAggregator.cs @@ -38,6 +38,8 @@ namespace CodexDistTestCore public void DownloadAllMetrics() { + var download = new MetricsDownloader(log, activePrometheuses); + download.DownloadAllMetrics(); } private string GeneratePrometheusConfig(OnlineCodexNode[] nodes) diff --git a/CodexDistTestCore/MetricsDownloader.cs b/CodexDistTestCore/MetricsDownloader.cs new file mode 100644 index 0000000..22ef526 --- /dev/null +++ b/CodexDistTestCore/MetricsDownloader.cs @@ -0,0 +1,97 @@ +using System.Globalization; + +namespace CodexDistTestCore +{ + public class MetricsDownloader + { + private readonly TestLog log; + private readonly Dictionary activePrometheuses; + + public MetricsDownloader(TestLog log, Dictionary activePrometheuses) + { + this.log = log; + this.activePrometheuses = activePrometheuses; + } + + public void DownloadAllMetrics() + { + foreach (var pair in activePrometheuses) + { + DownloadAllMetrics(pair.Key, pair.Value); + } + } + + private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes) + { + foreach (var node in nodes) + { + DownloadAllMetricsForNode(query, node); + } + } + + private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node) + { + var metrics = query.GetAllMetricsForNode(node); + if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return; + + var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); + var map = CreateValueMap(metrics); + + WriteToFile(node.GetName(), headers, map); + } + + private void WriteToFile(string nodeName, string[] headers, Dictionary> map) + { + var file = log.CreateSubfile("csv"); + log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}"); + + file.WriteRaw(string.Join(",", headers)); + + foreach (var pair in map) + { + file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value))); + } + } + + private Dictionary> CreateValueMap(Metrics metrics) + { + var map = CreateForAllTimestamps(metrics); + foreach (var metric in metrics.Sets) + { + AddToMap(map, metric); + } + return map; + + } + + private Dictionary> CreateForAllTimestamps(Metrics metrics) + { + var result = new Dictionary>(); + var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray(); + foreach (var timestamp in timestamps) result.Add(timestamp, new List()); + return result; + } + + private void AddToMap(Dictionary> map, MetricsSet metric) + { + foreach (var key in map.Keys) + { + map[key].Add(GetValueAtTimestamp(key, metric)); + } + } + + private string GetValueAtTimestamp(DateTime key, MetricsSet metric) + { + var value = metric.Values.SingleOrDefault(v => v.Timestamp == key); + if (value == null) return ""; + return value.Value.ToString(CultureInfo.InvariantCulture); + } + + private string FormatTimestamp(DateTime key) + { + var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + var diff = key - origin; + return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture); + } + } +} diff --git a/CodexDistTestCore/MetricsQuery.cs b/CodexDistTestCore/MetricsQuery.cs index 750aa52..d8ba684 100644 --- a/CodexDistTestCore/MetricsQuery.cs +++ b/CodexDistTestCore/MetricsQuery.cs @@ -16,9 +16,9 @@ namespace CodexDistTestCore "api/v1"); } - public Metrics? GetMostRecent(string metricName) + public Metrics? GetMostRecent(string metricName, OnlineCodexNode node) { - var response = GetLastOverTime(metricName); + var response = GetLastOverTime(metricName, GetInstanceStringForNode(node)); if (response == null) return null; return new Metrics @@ -38,23 +38,19 @@ namespace CodexDistTestCore { var response = GetAll(metricName); if (response == null) return null; - - return new Metrics - { - Sets = response.data.result.Select(r => - { - return new MetricsSet - { - Instance = r.metric.instance, - Values = MapMultipleValues(r.values) - }; - }).ToArray() - }; + return MapResponseToMetrics(response); } - private PrometheusQueryResponse? GetLastOverTime(string metricName) + public Metrics? GetAllMetricsForNode(OnlineCodexNode node) { - var response = http.HttpGetJson($"query?query=last_over_time({metricName}{GetQueryTimeRange()})"); + var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return MapResponseToMetrics(response); + } + + private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) + { + var response = http.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); if (response.status != "success") return null; return response; } @@ -66,6 +62,22 @@ namespace CodexDistTestCore return response; } + private Metrics MapResponseToMetrics(PrometheusQueryResponse response) + { + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Name = r.metric.__name__, + Instance = r.metric.instance, + Values = MapMultipleValues(r.values) + }; + }).ToArray() + }; + } + private MetricsSetValue[] MapSingleValue(object[] value) { if (value != null && value.Length > 0) @@ -98,6 +110,17 @@ namespace CodexDistTestCore }; } + private string GetInstanceNameForNode(OnlineCodexNode node) + { + var pod = node.Group.PodInfo!; + return $"{pod.Ip}:{node.Container.MetricsPort}"; + } + + private string GetInstanceStringForNode(OnlineCodexNode node) + { + return "{instance=\"" + GetInstanceNameForNode(node) + "\"}"; + } + private string GetQueryTimeRange() { return "[12h]"; @@ -122,6 +145,7 @@ namespace CodexDistTestCore public class MetricsSet { + public string Name { get; set; } = string.Empty; public string Instance { get; set; } = string.Empty; public MetricsSetValue[] Values { get; set; } = Array.Empty(); } @@ -157,4 +181,10 @@ namespace CodexDistTestCore public string instance { get; set; } = string.Empty; public string job { get; set; } = string.Empty; } + + public class PrometheusAllNamesResponse + { + public string status { get; set; } = string.Empty; + public string[] data { get; set; } = Array.Empty(); + } } diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index 658ba66..3947678 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -1,6 +1,5 @@ using CodexDistTestCore.Config; using NUnit.Framework; -using System.Xml.Linq; namespace CodexDistTestCore { @@ -52,9 +51,9 @@ namespace CodexDistTestCore file.ConcatToFilename("_FAILED"); } - public LogFile CreateSubfile() + public LogFile CreateSubfile(string ext = "log") { - return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}"); + return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); } private static string GetTestName() @@ -76,12 +75,14 @@ namespace CodexDistTestCore { private readonly DateTime now; private string name; + private readonly string ext; private readonly string filepath; - public LogFile(DateTime now, string name) + public LogFile(DateTime now, string name, string ext = "log") { this.now = now; this.name = name; + this.ext = ext; filepath = Path.Join( LogConfig.LogRoot, @@ -136,7 +137,7 @@ namespace CodexDistTestCore private void GenerateFilename() { - FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log"; + FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}"; FullFilename = Path.Combine(filepath, FilenameWithoutPath); } } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index cb5c0f1..9a8c5c8 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -53,6 +53,8 @@ namespace Tests.BasicTests primary.ConnectToPeer(secondary); primary2.ConnectToPeer(secondary2); + Thread.Sleep(TimeSpan.FromMinutes(5)); + metrics.AssertThat(primary, "libp2p_peers", Is.EqualTo(1)); metrics2.AssertThat(primary2, "libp2p_peers", Is.EqualTo(1)); } @@ -92,19 +94,19 @@ namespace Tests.BasicTests PerformTwoClientTest(primary, secondary); } - //[Test] - //public void TwoClientsTwoLocationsTest() - //{ - // var primary = SetupCodexNodes(1) - // .At(Location.BensLaptop) - // .BringOnline()[0]; + [Test] + public void TwoClientsTwoLocationsTest() + { + var primary = SetupCodexNodes(1) + .At(Location.BensLaptop) + .BringOnline()[0]; - // var secondary = SetupCodexNodes(1) - // .At(Location.BensOldGamingMachine) - // .BringOnline()[0]; + var secondary = SetupCodexNodes(1) + .At(Location.BensOldGamingMachine) + .BringOnline()[0]; - // PerformTwoClientTest(primary, secondary); - //} + PerformTwoClientTest(primary, secondary); + } private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) {