codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extracts query logic from metricsAccess object

This commit is contained in:
benbierens 2023-03-29 11:29:43 +02:00
parent f0d60493ae
commit 935c2320a7
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
3 changed files with 181 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
CodexDistTestCore/MetricsAccess.cs
View File

@ -1,5 +1,4 @@
using CodexDistTestCore.Config;
using NUnit.Framework;
using NUnit.Framework;
using NUnit.Framework.Constraints;
namespace CodexDistTestCore
@ -11,33 +10,34 @@ namespace CodexDistTestCore
public class MetricsAccess : IMetricsAccess
{
private readonly K8sCluster k8sCluster = new K8sCluster();
private readonly Http http;
private readonly MetricsQuery query;
private readonly OnlineCodexNode[] nodes;
public MetricsAccess(PrometheusInfo prometheusInfo, OnlineCodexNode[] nodes)
public MetricsAccess(MetricsQuery query, OnlineCodexNode[] nodes)
{
http = new Http(
k8sCluster.GetIp(),
prometheusInfo.ServicePort,
"api/v1");
this.query = query;
this.nodes = nodes;
}
public void AssertThat(IOnlineCodexNode node, string metricName, IResolveConstraint constraint, string message = "")
{
var metricValue = GetMetricWithTimeout(metricName, node);
var n = (OnlineCodexNode)node;
CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " +
"(This CodexNode is tracked by a different instance.)");
var metricSet = GetMetricWithTimeout(metricName, n);
var metricValue = metricSet.Values[0].Value;
Assert.That(metricValue, constraint, message);
}
private double GetMetricWithTimeout(string metricName, IOnlineCodexNode node)
private MetricsSet GetMetricWithTimeout(string metricName, OnlineCodexNode node)
{
var start = DateTime.UtcNow;
while (true)
{
var mostRecent = GetMostRecent(metricName, node);
if (mostRecent != null) return Convert.ToDouble(mostRecent);
if (mostRecent != null) return mostRecent;
if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
{
Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
@ -48,60 +48,14 @@ namespace CodexDistTestCore
}
}
private object? GetMostRecent(string metricName, IOnlineCodexNode node)
private MetricsSet? GetMostRecent(string metricName, OnlineCodexNode node)
{
var n = (OnlineCodexNode)node;
CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " +
"(This CodexNode is tracked by a different instance.)");
var result = query.GetMostRecent(metricName);
if (result == null) return null;
var response = GetMetric(metricName);
if (response == null) return null;
var value = GetValueFromResponse(n, response);
if (value == null) return null;
if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string].");
return value[1];
}
private PrometheusQueryResponse? GetMetric(string metricName)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}[12h])");
if (response.status != "success") return null;
return response;
}
private object[]? GetValueFromResponse(OnlineCodexNode node, PrometheusQueryResponse response)
{
var pod = node.Group.PodInfo!;
var forNode = response.data.result.SingleOrDefault(d => d.metric.instance == $"{pod.Ip}:{node.Container.MetricsPort}");
if (forNode == null) return null;
if (forNode.value == null || forNode.value.Length == 0) return null;
return forNode.value;
var instance = $"{pod.Ip}:{node.Container.MetricsPort}";
return result.Sets.SingleOrDefault(r => r.Instance == instance);
}
}
public class PrometheusQueryResponse
{
public string status { get; set; } = string.Empty;
public PrometheusQueryResponseData data { get; set; } = new();
}
public class PrometheusQueryResponseData
{
public string resultType { get; set; } = string.Empty;
public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty<PrometheusQueryResponseDataResultEntry>();
}
public class PrometheusQueryResponseDataResultEntry
{
public ResultEntryMetric metric { get; set; } = new();
public object[] value { get; set; } = Array.Empty<object>();
}
public class ResultEntryMetric
{
public string __name__ { get; set; } = string.Empty;
public string instance { get; set; } = string.Empty;
public string job { get; set; } = string.Empty;
}
}

8
CodexDistTestCore/MetricsAggregator.cs
View File

@ -8,7 +8,7 @@ namespace CodexDistTestCore
private readonly NumberSource prometheusNumberSource = new NumberSource(0);
private readonly TestLog log;
private readonly K8sManager k8sManager;
private readonly Dictionary<PrometheusInfo, OnlineCodexNode[]> activePrometheuses = new Dictionary<PrometheusInfo, OnlineCodexNode[]>();
private readonly Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses = new Dictionary<MetricsQuery, OnlineCodexNode[]>();
public MetricsAggregator(TestLog log, K8sManager k8sManager)
{
@ -29,10 +29,11 @@ namespace CodexDistTestCore
var config = GeneratePrometheusConfig(nodes);
var prometheus = k8sManager.BringOnlinePrometheus(config, prometheusNumberSource.GetNextNumber());
activePrometheuses.Add(prometheus, nodes);
var query = new MetricsQuery(prometheus);
activePrometheuses.Add(query, nodes);
log.Log("Metrics service started.");
return new MetricsAccess(prometheus, nodes);
return new MetricsAccess(query, nodes);
}
public void DownloadAllMetrics()
@ -51,7 +52,6 @@ namespace CodexDistTestCore
config += " metrics_path: /metrics\n";
config += " static_configs:\n";
config += " - targets:\n";
config += " - 'prometheus:9090'\n";
foreach (var node in nodes)
{

160
CodexDistTestCore/MetricsQuery.cs Normal file
View File

@ -0,0 +1,160 @@
using CodexDistTestCore.Config;
using System.Globalization;
namespace CodexDistTestCore
{
public class MetricsQuery
{
private readonly K8sCluster k8sCluster = new K8sCluster();
private readonly Http http;
public MetricsQuery(PrometheusInfo prometheusInfo)
{
http = new Http(
k8sCluster.GetIp(),
prometheusInfo.ServicePort,
"api/v1");
}
public Metrics? GetMostRecent(string metricName)
{
var response = GetLastOverTime(metricName);
if (response == null) return null;
return new Metrics
{
Sets = response.data.result.Select(r =>
{
return new MetricsSet
{
Instance = r.metric.instance,
Values = MapSingleValue(r.value)
};
}).ToArray()
};
}
public Metrics? GetMetrics(string metricName)
{
var response = GetAll(metricName);
if (response == null) return null;
return new Metrics
{
Sets = response.data.result.Select(r =>
{
return new MetricsSet
{
Instance = r.metric.instance,
Values = MapMultipleValues(r.values)
};
}).ToArray()
};
}
private PrometheusQueryResponse? GetLastOverTime(string metricName)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{GetQueryTimeRange()})");
if (response.status != "success") return null;
return response;
}
private PrometheusQueryResponse? GetAll(string metricName)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}");
if (response.status != "success") return null;
return response;
}
private MetricsSetValue[] MapSingleValue(object[] value)
{
if (value != null && value.Length > 0)
{
return new[]
{
MapValue(value)
};
}
return Array.Empty<MetricsSetValue>();
}
private MetricsSetValue[] MapMultipleValues(object[][] values)
{
if (values != null && values.Length > 0)
{
return values.Select(v => MapValue(v)).ToArray();
}
return Array.Empty<MetricsSetValue>();
}
private MetricsSetValue MapValue(object[] value)
{
if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string].");
return new MetricsSetValue
{
Timestamp = ToTimestamp(value[0]),
Value = ToValue(value[1])
};
}
private string GetQueryTimeRange()
{
return "[12h]";
}
private double ToValue(object v)
{
return Convert.ToDouble(v, CultureInfo.InvariantCulture);
}
private DateTime ToTimestamp(object v)
{
var unixSeconds = ToValue(v);
return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
}
}
public class Metrics
{
public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>();
}
public class MetricsSet
{
public string Instance { get; set; } = string.Empty;
public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
}
public class MetricsSetValue
{
public DateTime Timestamp { get; set; }
public double Value { get; set; }
}
public class PrometheusQueryResponse
{
public string status { get; set; } = string.Empty;
public PrometheusQueryResponseData data { get; set; } = new();
}
public class PrometheusQueryResponseData
{
public string resultType { get; set; } = string.Empty;
public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty<PrometheusQueryResponseDataResultEntry>();
}
public class PrometheusQueryResponseDataResultEntry
{
public ResultEntryMetric metric { get; set; } = new();
public object[] value { get; set; } = Array.Empty<object>();
public object[][] values { get; set; } = Array.Empty<object[]>();
}
public class ResultEntryMetric
{
public string __name__ { get; set; } = string.Empty;
public string instance { get; set; } = string.Empty;
public string job { get; set; } = string.Empty;
}
}