Metrics example test passes

This commit is contained in:
benbierens 2023-04-13 14:36:17 +02:00
parent 31e034ab67
commit 33a3f85136
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
19 changed files with 604 additions and 52 deletions

View File

@ -4,6 +4,8 @@ namespace DistTestCore.Codex
{
public class CodexContainerRecipe : ContainerRecipeFactory
{
public const string MetricsPortTag = "metrics_port";
protected override string Image => "thatbenbierens/nim-codex:sha-b204837";
protected override void Initialize(StartupConfig startupConfig)
@ -28,7 +30,7 @@ namespace DistTestCore.Codex
if (config.MetricsEnabled)
{
AddEnvVar("METRICS_ADDR", "0.0.0.0");
AddInternalPortAndVar("METRICS_PORT");
AddInternalPortAndVar("METRICS_PORT", tag: MetricsPortTag);
}
}
}

View File

@ -0,0 +1,28 @@
using DistTestCore.Codex;
using DistTestCore.Metrics;
namespace DistTestCore
{
public interface ICodexNodeFactory
{
OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group);
}
public class CodexNodeFactory : ICodexNodeFactory
{
private readonly TestLifecycle lifecycle;
private readonly IMetricsAccessFactory metricsAccessFactory;
public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory)
{
this.lifecycle = lifecycle;
this.metricsAccessFactory = metricsAccessFactory;
}
public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
{
var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container);
return new OnlineCodexNode(lifecycle, access, group, metricsAccess);
}
}
}

View File

@ -14,12 +14,12 @@ namespace DistTestCore
{
private readonly TestLifecycle lifecycle;
public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers)
public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory)
{
this.lifecycle = lifecycle;
Setup = setup;
Containers = containers;
Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c)).ToArray();
Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
}
public IOnlineCodexNode this[int index]
@ -73,14 +73,13 @@ namespace DistTestCore
public string Describe()
{
var orderNumber = Containers.RunningPod.Ip;
return $"CodexNodeGroup@{orderNumber}-{Setup.Describe()}";
return $"CodexNodeGroup@{Containers.Describe()}-{Setup.Describe()}";
}
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c)
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
{
var access = new CodexAccess(c);
return new OnlineCodexNode(lifecycle, access, this);
return factory.CreateOnlineCodexNode(access, this);
}
}
}

View File

@ -5,31 +5,27 @@ namespace DistTestCore
{
public class CodexStarter
{
private readonly WorkflowCreator workflowCreator;
private readonly TestLifecycle lifecycle;
private readonly WorkflowCreator workflowCreator;
public CodexStarter(TestLifecycle lifecycle, Configuration configuration)
public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
{
workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration());
this.lifecycle = lifecycle;
this.workflowCreator = workflowCreator;
}
public List<CodexNodeGroup> RunningGroups { get; } = new List<CodexNodeGroup>();
public ICodexNodeGroup BringOnline(CodexSetup codexSetup)
{
Log($"Starting {codexSetup.Describe()}...");
var containers = StartCodexContainers(codexSetup);
var workflow = CreateWorkflow();
var startupConfig = new StartupConfig();
startupConfig.Add(codexSetup);
var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers);
var runningContainers = workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig);
var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory);
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers);
RunningGroups.Add(group);
var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
Log($"Started at '{group.Containers.RunningPod.Ip}'");
return group;
}
@ -55,6 +51,26 @@ namespace DistTestCore
var workflow = CreateWorkflow();
workflow.DownloadContainerLog(container, logHandler);
}
private RunningContainers StartCodexContainers(CodexSetup codexSetup)
{
Log($"Starting {codexSetup.Describe()}...");
var workflow = CreateWorkflow();
var startupConfig = new StartupConfig();
startupConfig.Add(codexSetup);
return workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig);
}
private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory)
{
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
RunningGroups.Add(group);
Log($"Started at '{group.Containers.RunningPod.Ip}'");
return group;
}
private StartupWorkflow CreateWorkflow()
{

View File

@ -0,0 +1,65 @@
using KubernetesWorkflow;
using NUnit.Framework;
using NUnit.Framework.Constraints;
using Utils;
namespace DistTestCore.Metrics
{
public interface IMetricsAccess
{
void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
}
public class MetricsUnavailable : IMetricsAccess
{
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
{
Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
throw new InvalidOperationException();
}
}
public class MetricsAccess : IMetricsAccess
{
private readonly MetricsQuery query;
private readonly RunningContainer node;
public MetricsAccess(MetricsQuery query, RunningContainer node)
{
this.query = query;
this.node = node;
}
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
{
var metricSet = GetMetricWithTimeout(metricName);
var metricValue = metricSet.Values[0].Value;
Assert.That(metricValue, constraint, message);
}
private MetricsSet GetMetricWithTimeout(string metricName)
{
var start = DateTime.UtcNow;
while (true)
{
var mostRecent = GetMostRecent(metricName);
if (mostRecent != null) return mostRecent;
if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
{
Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
throw new TimeoutException();
}
Time.Sleep(TimeSpan.FromSeconds(2));
}
}
private MetricsSet? GetMostRecent(string metricName)
{
var result = query.GetMostRecent(metricName, node);
if (result == null) return null;
return result.Sets.LastOrDefault();
}
}
}

View File

@ -0,0 +1,33 @@
using KubernetesWorkflow;
namespace DistTestCore.Metrics
{
public interface IMetricsAccessFactory
{
IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer);
}
public class MetricsUnavailableAccessFactory : IMetricsAccessFactory
{
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
{
return new MetricsUnavailable();
}
}
public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory
{
private readonly RunningContainers prometheusContainer;
public CodexNodeMetricsAccessFactory(RunningContainers prometheusContainer)
{
this.prometheusContainer = prometheusContainer;
}
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
{
var query = new MetricsQuery(prometheusContainer);
return new MetricsAccess(query, codexContainer);
}
}
}

View File

@ -0,0 +1,98 @@
using Logging;
using System.Globalization;
namespace DistTestCore.Metrics
{
public class MetricsDownloader
{
private readonly TestLog log;
private readonly Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses;
public MetricsDownloader(TestLog log, Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses)
{
this.log = log;
this.activePrometheuses = activePrometheuses;
}
public void DownloadAllMetrics()
{
foreach (var pair in activePrometheuses)
{
DownloadAllMetrics(pair.Key, pair.Value);
}
}
private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes)
{
foreach (var node in nodes)
{
DownloadAllMetricsForNode(query, node);
}
}
private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node)
{
var metrics = query.GetAllMetricsForNode(node.CodexAccess.Container);
if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return;
var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray();
var map = CreateValueMap(metrics);
WriteToFile(node.GetName(), headers, map);
}
private void WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)
{
var file = log.CreateSubfile("csv");
log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}");
file.WriteRaw(string.Join(",", headers));
foreach (var pair in map)
{
file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
}
}
private Dictionary<DateTime, List<string>> CreateValueMap(Metrics metrics)
{
var map = CreateForAllTimestamps(metrics);
foreach (var metric in metrics.Sets)
{
AddToMap(map, metric);
}
return map;
}
private Dictionary<DateTime, List<string>> CreateForAllTimestamps(Metrics metrics)
{
var result = new Dictionary<DateTime, List<string>>();
var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray();
foreach (var timestamp in timestamps) result.Add(timestamp, new List<string>());
return result;
}
private void AddToMap(Dictionary<DateTime, List<string>> map, MetricsSet metric)
{
foreach (var key in map.Keys)
{
map[key].Add(GetValueAtTimestamp(key, metric));
}
}
private string GetValueAtTimestamp(DateTime key, MetricsSet metric)
{
var value = metric.Values.SingleOrDefault(v => v.Timestamp == key);
if (value == null) return "";
return value.Value.ToString(CultureInfo.InvariantCulture);
}
private string FormatTimestamp(DateTime key)
{
var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
var diff = key - origin;
return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture);
}
}
}

View File

@ -0,0 +1,195 @@
using DistTestCore.Codex;
using KubernetesWorkflow;
using System.Globalization;
namespace DistTestCore.Metrics
{
public class MetricsQuery
{
private readonly Http http;
public MetricsQuery(RunningContainers runningContainers)
{
RunningContainers = runningContainers;
http = new Http(
runningContainers.RunningPod.Cluster.IP,
runningContainers.Containers[0].ServicePorts[0].Number,
"api/v1");
}
public RunningContainers RunningContainers { get; }
public Metrics? GetMostRecent(string metricName, RunningContainer node)
{
var response = GetLastOverTime(metricName, GetInstanceStringForNode(node));
if (response == null) return null;
return new Metrics
{
Sets = response.data.result.Select(r =>
{
return new MetricsSet
{
Instance = r.metric.instance,
Values = MapSingleValue(r.value)
};
}).ToArray()
};
}
public Metrics? GetMetrics(string metricName)
{
var response = GetAll(metricName);
if (response == null) return null;
return MapResponseToMetrics(response);
}
public Metrics? GetAllMetricsForNode(RunningContainer node)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}");
if (response.status != "success") return null;
return MapResponseToMetrics(response);
}
private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})");
if (response.status != "success") return null;
return response;
}
private PrometheusQueryResponse? GetAll(string metricName)
{
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}");
if (response.status != "success") return null;
return response;
}
private Metrics MapResponseToMetrics(PrometheusQueryResponse response)
{
return new Metrics
{
Sets = response.data.result.Select(r =>
{
return new MetricsSet
{
Name = r.metric.__name__,
Instance = r.metric.instance,
Values = MapMultipleValues(r.values)
};
}).ToArray()
};
}
private MetricsSetValue[] MapSingleValue(object[] value)
{
if (value != null && value.Length > 0)
{
return new[]
{
MapValue(value)
};
}
return Array.Empty<MetricsSetValue>();
}
private MetricsSetValue[] MapMultipleValues(object[][] values)
{
if (values != null && values.Length > 0)
{
return values.Select(v => MapValue(v)).ToArray();
}
return Array.Empty<MetricsSetValue>();
}
private MetricsSetValue MapValue(object[] value)
{
if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string].");
return new MetricsSetValue
{
Timestamp = ToTimestamp(value[0]),
Value = ToValue(value[1])
};
}
private string GetInstanceNameForNode(RunningContainer node)
{
var ip = node.Pod.Ip;
var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
return $"{ip}:{port}";
}
private string GetInstanceStringForNode(RunningContainer node)
{
return "{instance=\"" + GetInstanceNameForNode(node) + "\"}";
}
private string GetQueryTimeRange()
{
return "[12h]";
}
private double ToValue(object v)
{
return Convert.ToDouble(v, CultureInfo.InvariantCulture);
}
private DateTime ToTimestamp(object v)
{
var unixSeconds = ToValue(v);
return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
}
}
public class Metrics
{
public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>();
}
public class MetricsSet
{
public string Name { get; set; } = string.Empty;
public string Instance { get; set; } = string.Empty;
public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
}
public class MetricsSetValue
{
public DateTime Timestamp { get; set; }
public double Value { get; set; }
}
public class PrometheusQueryResponse
{
public string status { get; set; } = string.Empty;
public PrometheusQueryResponseData data { get; set; } = new();
}
public class PrometheusQueryResponseData
{
public string resultType { get; set; } = string.Empty;
public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty<PrometheusQueryResponseDataResultEntry>();
}
public class PrometheusQueryResponseDataResultEntry
{
public ResultEntryMetric metric { get; set; } = new();
public object[] value { get; set; } = Array.Empty<object>();
public object[][] values { get; set; } = Array.Empty<object[]>();
}
public class ResultEntryMetric
{
public string __name__ { get; set; } = string.Empty;
public string instance { get; set; } = string.Empty;
public string job { get; set; } = string.Empty;
}
public class PrometheusAllNamesResponse
{
public string status { get; set; } = string.Empty;
public string[] data { get; set; } = Array.Empty<string>();
}
}

View File

@ -0,0 +1,17 @@
using KubernetesWorkflow;
namespace DistTestCore.Metrics
{
public class PrometheusContainerRecipe : ContainerRecipeFactory
{
protected override string Image => "thatbenbierens/prometheus-envconf:latest";
protected override void Initialize(StartupConfig startupConfig)
{
var config = startupConfig.Get<PrometheusStartupConfig>();
AddExposedPortAndVar("PROM_PORT");
AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
}
}
}

View File

@ -0,0 +1,12 @@
namespace DistTestCore.Metrics
{
public class PrometheusStartupConfig
{
public PrometheusStartupConfig(string prometheusConfigBase64)
{
PrometheusConfigBase64 = prometheusConfigBase64;
}
public string PrometheusConfigBase64 { get; }
}
}

View File

@ -1,5 +1,6 @@
using DistTestCore.Codex;
using DistTestCore.CodexLogsAndMetrics;
using DistTestCore.Metrics;
using NUnit.Framework;
namespace DistTestCore
@ -11,7 +12,7 @@ namespace DistTestCore
TestFile? DownloadContent(ContentId contentId);
void ConnectToPeer(IOnlineCodexNode node);
ICodexNodeLog DownloadLog();
//IMetricsAccess Metrics { get; }
IMetricsAccess Metrics { get; }
//IMarketplaceAccess Marketplace { get; }
}
@ -21,15 +22,17 @@ namespace DistTestCore
private const string UploadFailedMessage = "Unable to store block";
private readonly TestLifecycle lifecycle;
public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group)
public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess)
{
this.lifecycle = lifecycle;
CodexAccess = codexAccess;
Group = group;
Metrics = metricsAccess;
}
public CodexAccess CodexAccess { get; }
public CodexNodeGroup Group { get; }
public IMetricsAccess Metrics { get; }
public string GetName()
{

View File

@ -0,0 +1,65 @@
using DistTestCore.Codex;
using DistTestCore.Metrics;
using KubernetesWorkflow;
using System.Text;
namespace DistTestCore
{
public class PrometheusStarter
{
private readonly TestLifecycle lifecycle;
private readonly WorkflowCreator workflowCreator;
public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
{
this.lifecycle = lifecycle;
this.workflowCreator = workflowCreator;
}
public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers)
{
if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
Log($"Starting metrics server for {containers.Describe()}");
var startupConfig = new StartupConfig();
startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers)));
var workflow = workflowCreator.CreateWorkflow();
var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
Log("Metrics server started.");
return new CodexNodeMetricsAccessFactory(runningContainers);
}
private string GeneratePrometheusConfig(RunningContainer[] nodes)
{
var config = "";
config += "global:\n";
config += " scrape_interval: 30s\n";
config += " scrape_timeout: 10s\n";
config += "\n";
config += "scrape_configs:\n";
config += " - job_name: services\n";
config += " metrics_path: /metrics\n";
config += " static_configs:\n";
config += " - targets:\n";
foreach (var node in nodes)
{
var ip = node.Pod.Ip;
var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
config += $" - '{ip}:{port}'\n";
}
var bytes = Encoding.ASCII.GetBytes(config);
return Convert.ToBase64String(bytes);
}
private void Log(string msg)
{
lifecycle.Log.Log(msg);
}
}
}

View File

@ -1,20 +1,27 @@
using DistTestCore.CodexLogsAndMetrics;
using KubernetesWorkflow;
using Logging;
namespace DistTestCore
{
public class TestLifecycle
{
private readonly WorkflowCreator workflowCreator;
public TestLifecycle(Configuration configuration)
{
Log = new TestLog(configuration.GetLogConfig());
workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration());
FileManager = new FileManager(Log, configuration);
CodexStarter = new CodexStarter(this, configuration);
CodexStarter = new CodexStarter(this, workflowCreator);
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
}
public TestLog Log { get; }
public FileManager FileManager { get; }
public CodexStarter CodexStarter { get; }
public PrometheusStarter PrometheusStarter { get; }
public void DeleteAllResources()
{

View File

@ -17,16 +17,23 @@
public Port[] ExposedPorts { get; }
public Port[] InternalPorts { get; }
public EnvVar[] EnvVars { get; }
public Port GetPortByTag(string tag)
{
return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag);
}
}
public class Port
{
public Port(int number)
public Port(int number, string tag)
{
Number = number;
Tag = tag;
}
public int Number { get; }
public string Tag { get; }
}
public class EnvVar

View File

@ -28,28 +28,28 @@
protected int ContainerNumber { get; private set; } = 0;
protected abstract void Initialize(StartupConfig config);
protected Port AddExposedPort()
protected Port AddExposedPort(string tag = "")
{
var p = factory.CreatePort();
var p = factory.CreatePort(tag);
exposedPorts.Add(p);
return p;
}
protected Port AddInternalPort()
protected Port AddInternalPort(string tag = "")
{
var p = factory.CreatePort();
var p = factory.CreatePort(tag);
internalPorts.Add(p);
return p;
}
protected void AddExposedPortAndVar(string name)
protected void AddExposedPortAndVar(string name, string tag = "")
{
AddEnvVar(name, AddExposedPort());
AddEnvVar(name, AddExposedPort(tag));
}
protected void AddInternalPortAndVar(string name)
protected void AddInternalPortAndVar(string name, string tag = "")
{
AddEnvVar(name, AddInternalPort());
AddEnvVar(name, AddInternalPort(tag));
}
protected void AddEnvVar(string name, string value)

View File

@ -277,7 +277,7 @@ namespace KubernetesWorkflow
foreach (var port in recipe.ExposedPorts)
{
var servicePort = workflowNumberSource.GetServicePort();
usedPorts.Add(new Port(servicePort));
usedPorts.Add(new Port(servicePort, ""));
result.Add(new V1ServicePort
{

View File

@ -7,9 +7,9 @@ namespace KubernetesWorkflow
{
private NumberSource portNumberSource = new NumberSource(8080);
public Port CreatePort()
public Port CreatePort(string tag)
{
return new Port(portNumberSource.GetNextNumber());
return new Port(portNumberSource.GetNextNumber(), tag);
}
public EnvVar CreateEnvVar(string name, int value)

View File

@ -12,6 +12,11 @@
public StartupConfig StartupConfig { get; }
public RunningPod RunningPod { get; }
public RunningContainer[] Containers { get; }
public string Describe()
{
return $"[{RunningPod.Ip}]";
}
}
public class RunningContainer

View File

@ -78,30 +78,30 @@ namespace Tests.BasicTests
log.AssertLogContains("Uploaded file");
}
//[Test]
//public void TwoMetricsExample()
//{
// var group = SetupCodexNodes(2)
// .EnableMetrics()
// .BringOnline();
[Test]
public void TwoMetricsExample()
{
var group = SetupCodexNodes(2)
.EnableMetrics()
.BringOnline();
// var group2 = SetupCodexNodes(2)
// .EnableMetrics()
// .BringOnline();
var group2 = SetupCodexNodes(2)
.EnableMetrics()
.BringOnline();
// var primary = group[0];
// var secondary = group[1];
// var primary2 = group2[0];
// var secondary2 = group2[1];
var primary = group[0];
var secondary = group[1];
var primary2 = group2[0];
var secondary2 = group2[1];
// primary.ConnectToPeer(secondary);
// primary2.ConnectToPeer(secondary2);
primary.ConnectToPeer(secondary);
primary2.ConnectToPeer(secondary2);
// Thread.Sleep(TimeSpan.FromMinutes(5));
Thread.Sleep(TimeSpan.FromMinutes(5));
// primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
// primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
//}
primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
}
//[Test]
//public void MarketplaceExample()