status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

implements downloading of metrics on test failure

This commit is contained in:
benbierens 2023-03-30 10:43:17 +02:00
parent 935c2320a7
commit f70ce8e8bb
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
6 changed files with 166 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CodexDistTestCore/MetricsAccess.cs
View File

@ -50,12 +50,9 @@ namespace CodexDistTestCore
private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node)
{
var result = query.GetMostRecent(metricName);
var result = query.GetMostRecent(metricName, node);
if (result == null) return null;
var pod = node.Group.PodInfo!;
var instance = $"{pod.Ip}:{node.Container.MetricsPort}";
return result.Sets.SingleOrDefault(r => r.Instance == instance);
return result.Sets.LastOrDefault();
}
}
}

2
CodexDistTestCore/MetricsAggregator.cs
View File

@ -38,6 +38,8 @@ namespace CodexDistTestCore
public void DownloadAllMetrics()
{
var download = new MetricsDownloader(log, activePrometheuses);
download.DownloadAllMetrics();
}
private string GeneratePrometheusConfig(OnlineCodexNode[] nodes)

97
CodexDistTestCore/MetricsDownloader.cs Normal file
View File

@ -0,0 +1,97 @@
using System.Globalization;
namespace CodexDistTestCore
{
public class MetricsDownloader
{
private readonly TestLog log;
private readonly Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses;
public MetricsDownloader(TestLog log, Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses)
{
this.log = log;
this.activePrometheuses = activePrometheuses;
}
public void DownloadAllMetrics()
{
foreach (var pair in activePrometheuses)
{
DownloadAllMetrics(pair.Key, pair.Value);
}
}
private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes)
{
foreach (var node in nodes)
{
DownloadAllMetricsForNode(query, node);
}
}
private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node)
{
var metrics = query.GetAllMetricsForNode(node);
if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return;
var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray();
var map = CreateValueMap(metrics);
WriteToFile(node.GetName(), headers, map);
}
private void WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)
{
var file = log.CreateSubfile("csv");
log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}");
file.WriteRaw(string.Join(",", headers));
foreach (var pair in map)
{
file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
}
}
private Dictionary<DateTime, List<string>> CreateValueMap(Metrics metrics)
{
var map = CreateForAllTimestamps(metrics);
foreach (var metric in metrics.Sets)
{
AddToMap(map, metric);
}
return map;
}
private Dictionary<DateTime, List<string>> CreateForAllTimestamps(Metrics metrics)
{
var result = new Dictionary<DateTime, List<string>>();
var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray();
foreach (var timestamp in timestamps) result.Add(timestamp, new List<string>());
return result;
}
private void AddToMap(Dictionary<DateTime, List<string>> map, MetricsSet metric)
{
foreach (var key in map.Keys)
{
map[key].Add(GetValueAtTimestamp(key, metric));
}
}
private string GetValueAtTimestamp(DateTime key, MetricsSet metric)
{
var value = metric.Values.SingleOrDefault(v => v.Timestamp == key);
if (value == null) return "";
return value.Value.ToString(CultureInfo.InvariantCulture);
}
private string FormatTimestamp(DateTime key)
{
var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
var diff = key - origin;
return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture);
}
}
}

62
CodexDistTestCore/MetricsQuery.cs
View File

@ -16,9 +16,9 @@ namespace CodexDistTestCore
"api/v1");
}
public Metrics? GetMostRecent(string metricName)
public Metrics? GetMostRecent(string metricName, OnlineCodexNode node)
{
var response = GetLastOverTime(metricName);
var response = GetLastOverTime(metricName, GetInstanceStringForNode(node));
if (response == null) return null;
return new Metrics
@ -38,23 +38,19 @@ namespace CodexDistTestCore
{
var response = GetAll(metricName);
if (response == null) return null;
return new Metrics
{
Sets = response.data.result.Select(r =>
{
return new MetricsSet
{
Instance = r.metric.instance,
Values = MapMultipleValues(r.values)
};
}).ToArray()
};
return MapResponseToMetrics(response);
}
private PrometheusQueryResponse? GetLastOverTime(string metricName)
public Metrics? GetAllMetricsForNode(OnlineCodexNode node)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{GetQueryTimeRange()})");
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}");
if (response.status != "success") return null;
return MapResponseToMetrics(response);
}
private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})");
if (response.status != "success") return null;
return response;
}
@ -66,6 +62,22 @@ namespace CodexDistTestCore
return response;
}
private Metrics MapResponseToMetrics(PrometheusQueryResponse response)
{
return new Metrics
{
Sets = response.data.result.Select(r =>
{
return new MetricsSet
{
Name = r.metric.__name__,
Instance = r.metric.instance,
Values = MapMultipleValues(r.values)
};
}).ToArray()
};
}
private MetricsSetValue[] MapSingleValue(object[] value)
{
if (value != null && value.Length > 0)
@ -98,6 +110,17 @@ namespace CodexDistTestCore
};
}
private string GetInstanceNameForNode(OnlineCodexNode node)
{
var pod = node.Group.PodInfo!;
return $"{pod.Ip}:{node.Container.MetricsPort}";
}
private string GetInstanceStringForNode(OnlineCodexNode node)
{
return "{instance=\"" + GetInstanceNameForNode(node) + "\"}";
}
private string GetQueryTimeRange()
{
return "[12h]";
@ -122,6 +145,7 @@ namespace CodexDistTestCore
public class MetricsSet
{
public string Name { get; set; } = string.Empty;
public string Instance { get; set; } = string.Empty;
public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
}
@ -157,4 +181,10 @@ namespace CodexDistTestCore
public string instance { get; set; } = string.Empty;
public string job { get; set; } = string.Empty;
}
public class PrometheusAllNamesResponse
{
public string status { get; set; } = string.Empty;
public string[] data { get; set; } = Array.Empty<string>();
}
}

11
CodexDistTestCore/TestLog.cs
View File

@ -1,6 +1,5 @@
using CodexDistTestCore.Config;
using NUnit.Framework;
using System.Xml.Linq;
namespace CodexDistTestCore
{
@ -52,9 +51,9 @@ namespace CodexDistTestCore
file.ConcatToFilename("_FAILED");
}
public LogFile CreateSubfile()
public LogFile CreateSubfile(string ext = "log")
{
return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}");
return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext);
}
private static string GetTestName()
@ -76,12 +75,14 @@ namespace CodexDistTestCore
{
private readonly DateTime now;
private string name;
private readonly string ext;
private readonly string filepath;
public LogFile(DateTime now, string name)
public LogFile(DateTime now, string name, string ext = "log")
{
this.now = now;
this.name = name;
this.ext = ext;
filepath = Path.Join(
LogConfig.LogRoot,
@ -136,7 +137,7 @@ namespace CodexDistTestCore
private void GenerateFilename()
{
FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log";
FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.{ext}";
FullFilename = Path.Combine(filepath, FilenameWithoutPath);
}
}

24
Tests/BasicTests/SimpleTests.cs
View File

@ -53,6 +53,8 @@ namespace Tests.BasicTests
primary.ConnectToPeer(secondary);
primary2.ConnectToPeer(secondary2);
Thread.Sleep(TimeSpan.FromMinutes(5));
metrics.AssertThat(primary, "libp2p_peers", Is.EqualTo(1));
metrics2.AssertThat(primary2, "libp2p_peers", Is.EqualTo(1));
}
@ -92,19 +94,19 @@ namespace Tests.BasicTests
PerformTwoClientTest(primary, secondary);
}
//[Test]
//public void TwoClientsTwoLocationsTest()
//{
// var primary = SetupCodexNodes(1)
// .At(Location.BensLaptop)
// .BringOnline()[0];
[Test]
public void TwoClientsTwoLocationsTest()
{
var primary = SetupCodexNodes(1)
.At(Location.BensLaptop)
.BringOnline()[0];
// var secondary = SetupCodexNodes(1)
// .At(Location.BensOldGamingMachine)
// .BringOnline()[0];
var secondary = SetupCodexNodes(1)
.At(Location.BensOldGamingMachine)
.BringOnline()[0];
// PerformTwoClientTest(primary, secondary);
//}
PerformTwoClientTest(primary, secondary);
}
private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
{