status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Simplifies node-metrics interface

This commit is contained in:
benbierens 2023-03-31 10:00:44 +02:00
parent f70ce8e8bb
commit c50cf02f7d
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
7 changed files with 55 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CodexDistTestCore/CodexNodeLog.cs
View File

@ -2,7 +2,12 @@
namespace CodexDistTestCore
{
public class CodexNodeLog
public interface ICodexNodeLog
{
void AssertLogContains(string expectedString);
}
public class CodexNodeLog : ICodexNodeLog
{
private readonly LogFile logFile;

20
CodexDistTestCore/DistTest.cs
View File

@ -9,7 +9,6 @@ namespace CodexDistTestCore
private TestLog log = null!;
private FileManager fileManager = null!;
private K8sManager k8sManager = null!;
private MetricsAggregator metricsAggregator = null!;
[OneTimeSetUp]
public void GlobalSetup()
@ -49,7 +48,6 @@ namespace CodexDistTestCore
fileManager = new FileManager(log);
k8sManager = new K8sManager(log, fileManager);
metricsAggregator = new MetricsAggregator(log, k8sManager);
}
}
@ -80,22 +78,6 @@ namespace CodexDistTestCore
return new OfflineCodexNodes(k8sManager, numberOfNodes);
}
public MetricsAccess GatherMetrics(ICodexNodeGroup group)
{
return GatherMetrics(group.ToArray());
}
public MetricsAccess GatherMetrics(params IOnlineCodexNode[] nodes)
{
var onlineNodes = nodes.Cast<OnlineCodexNode>().ToArray();
Assert.That(onlineNodes.All(n => n.Group.Origin.MetricsEnabled),
"Incorrect test setup: Metrics were not enabled on (all) provided OnlineCodexNodes. " +
"To use metrics, please use 'EnableMetrics()' when setting up Codex nodes.");
return metricsAggregator.BeginCollectingMetricsFor(onlineNodes);
}
private void IncludeLogsAndMetricsOnTestFailure()
{
var result = TestContext.CurrentContext.Result;
@ -105,7 +87,7 @@ namespace CodexDistTestCore
{
log.Log("Downloading all CodexNode logs and metrics because of test failure...");
k8sManager.ForEachOnlineGroup(DownloadLogs);
metricsAggregator.DownloadAllMetrics();
k8sManager.DownloadAllMetrics();
}
else
{

19
CodexDistTestCore/K8sManager.cs
View File

@ -14,11 +14,13 @@
private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly TestLog log;
private readonly IFileManager fileManager;
private readonly MetricsAggregator metricsAggregator;
public K8sManager(TestLog log, IFileManager fileManager)
{
this.log = log;
this.fileManager = fileManager;
metricsAggregator = new MetricsAggregator(log, this);
}
public ICodexNodeGroup BringOnline(OfflineCodexNodes offline)
@ -29,6 +31,11 @@
log.Log($"{online.Describe()} online.");
if (offline.MetricsEnabled)
{
BringOnlineMetrics(online);
}
return online;
}
@ -67,6 +74,18 @@
return info!;
}
public void DownloadAllMetrics()
{
metricsAggregator.DownloadAllMetrics();
}
private void BringOnlineMetrics(CodexNodeGroup group)
{
var onlineNodes = group.Nodes.Cast<OnlineCodexNode>().ToArray();
metricsAggregator.BeginCollectingMetricsFor(onlineNodes);
}
private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
{
var containers = CreateContainers(offline);

25
CodexDistTestCore/MetricsAccess.cs
View File

@ -5,27 +5,32 @@ namespace CodexDistTestCore
{
public interface IMetricsAccess
{
void AssertThat(IOnlineCodexNode node, string metricName, IResolveConstraint constraint, string message = "");
void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
}
public class MetricsUnavailable : IMetricsAccess
{
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
{
Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
throw new InvalidOperationException();
}
}
public class MetricsAccess : IMetricsAccess
{
private readonly MetricsQuery query;
private readonly OnlineCodexNode[] nodes;
private readonly OnlineCodexNode node;
public MetricsAccess(MetricsQuery query, OnlineCodexNode[] nodes)
public MetricsAccess(MetricsQuery query, OnlineCodexNode node)
{
this.query = query;
this.nodes = nodes;
this.node = node;
}
public void AssertThat(IOnlineCodexNode node, string metricName, IResolveConstraint constraint, string message = "")
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
{
var n = (OnlineCodexNode)node;
CollectionAssert.Contains(nodes, n, "Incorrect test setup: Attempt to get metrics for OnlineCodexNode from the wrong MetricsAccess object. " +
"(This CodexNode is tracked by a different instance.)");
var metricSet = GetMetricWithTimeout(metricName, n);
var metricSet = GetMetricWithTimeout(metricName, node);
var metricValue = metricSet.Values[0].Value;
Assert.That(metricValue, constraint, message);
}

15
CodexDistTestCore/MetricsAggregator.cs
View File

@ -16,15 +16,8 @@ namespace CodexDistTestCore
this.k8sManager = k8sManager;
}
public MetricsAccess BeginCollectingMetricsFor(OnlineCodexNode[] nodes)
public void BeginCollectingMetricsFor(OnlineCodexNode[] nodes)
{
var alreadyStartedNodes = nodes.Where(n => activePrometheuses.Values.Any(v => v.Contains(n)));
if (alreadyStartedNodes.Any())
{
Assert.Fail("Incorrect test setup: 'GatherMetrics' was already called on one or more of these OnlineCodexNodes.");
throw new InvalidOperationException();
}
log.Log($"Starting metrics collecting for {nodes.Length} nodes...");
var config = GeneratePrometheusConfig(nodes);
@ -33,7 +26,11 @@ namespace CodexDistTestCore
activePrometheuses.Add(query, nodes);
log.Log("Metrics service started.");
return new MetricsAccess(query, nodes);
foreach(var node in nodes)
{
node.Metrics = new MetricsAccess(query, node);
}
}
public void DownloadAllMetrics()

6
CodexDistTestCore/OnlineCodexNode.cs
View File

@ -9,7 +9,8 @@ namespace CodexDistTestCore
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId);
void ConnectToPeer(IOnlineCodexNode node);
CodexNodeLog DownloadLog();
ICodexNodeLog DownloadLog();
IMetricsAccess Metrics { get; }
}
public class OnlineCodexNode : IOnlineCodexNode
@ -30,6 +31,7 @@ namespace CodexDistTestCore
public CodexNodeContainer Container { get; }
public CodexNodeGroup Group { get; internal set; } = null!;
public IMetricsAccess Metrics { get; set; } = new MetricsUnavailable();
public string GetName()
{
@ -80,7 +82,7 @@ namespace CodexDistTestCore
Log($"Successfully connected to peer {peer.GetName()}.");
}
public CodexNodeLog DownloadLog()
public ICodexNodeLog DownloadLog()
{
return Group.DownloadLog(this);
}

12
Tests/BasicTests/SimpleTests.cs
View File

@ -37,13 +37,9 @@ namespace Tests.BasicTests
.EnableMetrics()
.BringOnline();
var metrics = GatherMetrics(group);
var group2 = SetupCodexNodes(2)
.EnableMetrics()
.BringOnline();
var metrics2 = GatherMetrics(group2);
.EnableMetrics()
.BringOnline();
var primary = group[0];
var secondary = group[1];
@ -55,8 +51,8 @@ namespace Tests.BasicTests
Thread.Sleep(TimeSpan.FromMinutes(5));
metrics.AssertThat(primary, "libp2p_peers", Is.EqualTo(1));
metrics2.AssertThat(primary2, "libp2p_peers", Is.EqualTo(1));
primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
}
[Test]