Metrics example test passes
This commit is contained in:
parent
31e034ab67
commit
33a3f85136
|
@ -4,6 +4,8 @@ namespace DistTestCore.Codex
|
|||
{
|
||||
public class CodexContainerRecipe : ContainerRecipeFactory
|
||||
{
|
||||
public const string MetricsPortTag = "metrics_port";
|
||||
|
||||
protected override string Image => "thatbenbierens/nim-codex:sha-b204837";
|
||||
|
||||
protected override void Initialize(StartupConfig startupConfig)
|
||||
|
@ -28,7 +30,7 @@ namespace DistTestCore.Codex
|
|||
if (config.MetricsEnabled)
|
||||
{
|
||||
AddEnvVar("METRICS_ADDR", "0.0.0.0");
|
||||
AddInternalPortAndVar("METRICS_PORT");
|
||||
AddInternalPortAndVar("METRICS_PORT", tag: MetricsPortTag);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,28 @@
|
|||
using DistTestCore.Codex;
|
||||
using DistTestCore.Metrics;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public interface ICodexNodeFactory
|
||||
{
|
||||
OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group);
|
||||
}
|
||||
|
||||
public class CodexNodeFactory : ICodexNodeFactory
|
||||
{
|
||||
private readonly TestLifecycle lifecycle;
|
||||
private readonly IMetricsAccessFactory metricsAccessFactory;
|
||||
|
||||
public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory)
|
||||
{
|
||||
this.lifecycle = lifecycle;
|
||||
this.metricsAccessFactory = metricsAccessFactory;
|
||||
}
|
||||
|
||||
public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
|
||||
{
|
||||
var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container);
|
||||
return new OnlineCodexNode(lifecycle, access, group, metricsAccess);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -14,12 +14,12 @@ namespace DistTestCore
|
|||
{
|
||||
private readonly TestLifecycle lifecycle;
|
||||
|
||||
public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers)
|
||||
public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory)
|
||||
{
|
||||
this.lifecycle = lifecycle;
|
||||
Setup = setup;
|
||||
Containers = containers;
|
||||
Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c)).ToArray();
|
||||
Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
|
||||
}
|
||||
|
||||
public IOnlineCodexNode this[int index]
|
||||
|
@ -73,14 +73,13 @@ namespace DistTestCore
|
|||
|
||||
public string Describe()
|
||||
{
|
||||
var orderNumber = Containers.RunningPod.Ip;
|
||||
return $"CodexNodeGroup@{orderNumber}-{Setup.Describe()}";
|
||||
return $"CodexNodeGroup@{Containers.Describe()}-{Setup.Describe()}";
|
||||
}
|
||||
|
||||
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c)
|
||||
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
|
||||
{
|
||||
var access = new CodexAccess(c);
|
||||
return new OnlineCodexNode(lifecycle, access, this);
|
||||
return factory.CreateOnlineCodexNode(access, this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -5,31 +5,27 @@ namespace DistTestCore
|
|||
{
|
||||
public class CodexStarter
|
||||
{
|
||||
private readonly WorkflowCreator workflowCreator;
|
||||
private readonly TestLifecycle lifecycle;
|
||||
private readonly WorkflowCreator workflowCreator;
|
||||
|
||||
public CodexStarter(TestLifecycle lifecycle, Configuration configuration)
|
||||
public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
|
||||
{
|
||||
workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration());
|
||||
this.lifecycle = lifecycle;
|
||||
this.workflowCreator = workflowCreator;
|
||||
}
|
||||
|
||||
public List<CodexNodeGroup> RunningGroups { get; } = new List<CodexNodeGroup>();
|
||||
|
||||
public ICodexNodeGroup BringOnline(CodexSetup codexSetup)
|
||||
{
|
||||
Log($"Starting {codexSetup.Describe()}...");
|
||||
var containers = StartCodexContainers(codexSetup);
|
||||
|
||||
var workflow = CreateWorkflow();
|
||||
var startupConfig = new StartupConfig();
|
||||
startupConfig.Add(codexSetup);
|
||||
var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers);
|
||||
|
||||
var runningContainers = workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig);
|
||||
var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory);
|
||||
|
||||
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers);
|
||||
RunningGroups.Add(group);
|
||||
var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
|
||||
|
||||
Log($"Started at '{group.Containers.RunningPod.Ip}'");
|
||||
return group;
|
||||
}
|
||||
|
||||
|
@ -55,6 +51,26 @@ namespace DistTestCore
|
|||
var workflow = CreateWorkflow();
|
||||
workflow.DownloadContainerLog(container, logHandler);
|
||||
}
|
||||
|
||||
private RunningContainers StartCodexContainers(CodexSetup codexSetup)
|
||||
{
|
||||
Log($"Starting {codexSetup.Describe()}...");
|
||||
|
||||
var workflow = CreateWorkflow();
|
||||
var startupConfig = new StartupConfig();
|
||||
startupConfig.Add(codexSetup);
|
||||
|
||||
return workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig);
|
||||
}
|
||||
|
||||
private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory)
|
||||
{
|
||||
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
|
||||
RunningGroups.Add(group);
|
||||
|
||||
Log($"Started at '{group.Containers.RunningPod.Ip}'");
|
||||
return group;
|
||||
}
|
||||
|
||||
private StartupWorkflow CreateWorkflow()
|
||||
{
|
||||
|
|
|
@ -0,0 +1,65 @@
|
|||
using KubernetesWorkflow;
|
||||
using NUnit.Framework;
|
||||
using NUnit.Framework.Constraints;
|
||||
using Utils;
|
||||
|
||||
namespace DistTestCore.Metrics
|
||||
{
|
||||
public interface IMetricsAccess
|
||||
{
|
||||
void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
|
||||
}
|
||||
|
||||
public class MetricsUnavailable : IMetricsAccess
|
||||
{
|
||||
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
|
||||
{
|
||||
Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
|
||||
throw new InvalidOperationException();
|
||||
}
|
||||
}
|
||||
|
||||
public class MetricsAccess : IMetricsAccess
|
||||
{
|
||||
private readonly MetricsQuery query;
|
||||
private readonly RunningContainer node;
|
||||
|
||||
public MetricsAccess(MetricsQuery query, RunningContainer node)
|
||||
{
|
||||
this.query = query;
|
||||
this.node = node;
|
||||
}
|
||||
|
||||
public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
|
||||
{
|
||||
var metricSet = GetMetricWithTimeout(metricName);
|
||||
var metricValue = metricSet.Values[0].Value;
|
||||
Assert.That(metricValue, constraint, message);
|
||||
}
|
||||
|
||||
private MetricsSet GetMetricWithTimeout(string metricName)
|
||||
{
|
||||
var start = DateTime.UtcNow;
|
||||
|
||||
while (true)
|
||||
{
|
||||
var mostRecent = GetMostRecent(metricName);
|
||||
if (mostRecent != null) return mostRecent;
|
||||
if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
|
||||
{
|
||||
Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
|
||||
throw new TimeoutException();
|
||||
}
|
||||
|
||||
Time.Sleep(TimeSpan.FromSeconds(2));
|
||||
}
|
||||
}
|
||||
|
||||
private MetricsSet? GetMostRecent(string metricName)
|
||||
{
|
||||
var result = query.GetMostRecent(metricName, node);
|
||||
if (result == null) return null;
|
||||
return result.Sets.LastOrDefault();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,33 @@
|
|||
using KubernetesWorkflow;
|
||||
|
||||
namespace DistTestCore.Metrics
|
||||
{
|
||||
public interface IMetricsAccessFactory
|
||||
{
|
||||
IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer);
|
||||
}
|
||||
|
||||
public class MetricsUnavailableAccessFactory : IMetricsAccessFactory
|
||||
{
|
||||
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
|
||||
{
|
||||
return new MetricsUnavailable();
|
||||
}
|
||||
}
|
||||
|
||||
public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory
|
||||
{
|
||||
private readonly RunningContainers prometheusContainer;
|
||||
|
||||
public CodexNodeMetricsAccessFactory(RunningContainers prometheusContainer)
|
||||
{
|
||||
this.prometheusContainer = prometheusContainer;
|
||||
}
|
||||
|
||||
public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
|
||||
{
|
||||
var query = new MetricsQuery(prometheusContainer);
|
||||
return new MetricsAccess(query, codexContainer);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,98 @@
|
|||
using Logging;
|
||||
using System.Globalization;
|
||||
|
||||
namespace DistTestCore.Metrics
|
||||
{
|
||||
public class MetricsDownloader
|
||||
{
|
||||
private readonly TestLog log;
|
||||
private readonly Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses;
|
||||
|
||||
public MetricsDownloader(TestLog log, Dictionary<MetricsQuery, OnlineCodexNode[]> activePrometheuses)
|
||||
{
|
||||
this.log = log;
|
||||
this.activePrometheuses = activePrometheuses;
|
||||
}
|
||||
|
||||
public void DownloadAllMetrics()
|
||||
{
|
||||
foreach (var pair in activePrometheuses)
|
||||
{
|
||||
DownloadAllMetrics(pair.Key, pair.Value);
|
||||
}
|
||||
}
|
||||
|
||||
private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes)
|
||||
{
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
DownloadAllMetricsForNode(query, node);
|
||||
}
|
||||
}
|
||||
|
||||
private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node)
|
||||
{
|
||||
var metrics = query.GetAllMetricsForNode(node.CodexAccess.Container);
|
||||
if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return;
|
||||
|
||||
var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray();
|
||||
var map = CreateValueMap(metrics);
|
||||
|
||||
WriteToFile(node.GetName(), headers, map);
|
||||
}
|
||||
|
||||
private void WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)
|
||||
{
|
||||
var file = log.CreateSubfile("csv");
|
||||
log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}");
|
||||
|
||||
file.WriteRaw(string.Join(",", headers));
|
||||
|
||||
foreach (var pair in map)
|
||||
{
|
||||
file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
|
||||
}
|
||||
}
|
||||
|
||||
private Dictionary<DateTime, List<string>> CreateValueMap(Metrics metrics)
|
||||
{
|
||||
var map = CreateForAllTimestamps(metrics);
|
||||
foreach (var metric in metrics.Sets)
|
||||
{
|
||||
AddToMap(map, metric);
|
||||
}
|
||||
return map;
|
||||
|
||||
}
|
||||
|
||||
private Dictionary<DateTime, List<string>> CreateForAllTimestamps(Metrics metrics)
|
||||
{
|
||||
var result = new Dictionary<DateTime, List<string>>();
|
||||
var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray();
|
||||
foreach (var timestamp in timestamps) result.Add(timestamp, new List<string>());
|
||||
return result;
|
||||
}
|
||||
|
||||
private void AddToMap(Dictionary<DateTime, List<string>> map, MetricsSet metric)
|
||||
{
|
||||
foreach (var key in map.Keys)
|
||||
{
|
||||
map[key].Add(GetValueAtTimestamp(key, metric));
|
||||
}
|
||||
}
|
||||
|
||||
private string GetValueAtTimestamp(DateTime key, MetricsSet metric)
|
||||
{
|
||||
var value = metric.Values.SingleOrDefault(v => v.Timestamp == key);
|
||||
if (value == null) return "";
|
||||
return value.Value.ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private string FormatTimestamp(DateTime key)
|
||||
{
|
||||
var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
|
||||
var diff = key - origin;
|
||||
return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,195 @@
|
|||
using DistTestCore.Codex;
|
||||
using KubernetesWorkflow;
|
||||
using System.Globalization;
|
||||
|
||||
namespace DistTestCore.Metrics
|
||||
{
|
||||
public class MetricsQuery
|
||||
{
|
||||
private readonly Http http;
|
||||
|
||||
public MetricsQuery(RunningContainers runningContainers)
|
||||
{
|
||||
RunningContainers = runningContainers;
|
||||
|
||||
http = new Http(
|
||||
runningContainers.RunningPod.Cluster.IP,
|
||||
runningContainers.Containers[0].ServicePorts[0].Number,
|
||||
"api/v1");
|
||||
}
|
||||
|
||||
public RunningContainers RunningContainers { get; }
|
||||
|
||||
public Metrics? GetMostRecent(string metricName, RunningContainer node)
|
||||
{
|
||||
var response = GetLastOverTime(metricName, GetInstanceStringForNode(node));
|
||||
if (response == null) return null;
|
||||
|
||||
return new Metrics
|
||||
{
|
||||
Sets = response.data.result.Select(r =>
|
||||
{
|
||||
return new MetricsSet
|
||||
{
|
||||
Instance = r.metric.instance,
|
||||
Values = MapSingleValue(r.value)
|
||||
};
|
||||
}).ToArray()
|
||||
};
|
||||
}
|
||||
|
||||
public Metrics? GetMetrics(string metricName)
|
||||
{
|
||||
var response = GetAll(metricName);
|
||||
if (response == null) return null;
|
||||
return MapResponseToMetrics(response);
|
||||
}
|
||||
|
||||
public Metrics? GetAllMetricsForNode(RunningContainer node)
|
||||
{
|
||||
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}");
|
||||
if (response.status != "success") return null;
|
||||
return MapResponseToMetrics(response);
|
||||
}
|
||||
|
||||
private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
|
||||
{
|
||||
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})");
|
||||
if (response.status != "success") return null;
|
||||
return response;
|
||||
}
|
||||
|
||||
private PrometheusQueryResponse? GetAll(string metricName)
|
||||
{
|
||||
var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}");
|
||||
if (response.status != "success") return null;
|
||||
return response;
|
||||
}
|
||||
|
||||
private Metrics MapResponseToMetrics(PrometheusQueryResponse response)
|
||||
{
|
||||
return new Metrics
|
||||
{
|
||||
Sets = response.data.result.Select(r =>
|
||||
{
|
||||
return new MetricsSet
|
||||
{
|
||||
Name = r.metric.__name__,
|
||||
Instance = r.metric.instance,
|
||||
Values = MapMultipleValues(r.values)
|
||||
};
|
||||
}).ToArray()
|
||||
};
|
||||
}
|
||||
|
||||
private MetricsSetValue[] MapSingleValue(object[] value)
|
||||
{
|
||||
if (value != null && value.Length > 0)
|
||||
{
|
||||
return new[]
|
||||
{
|
||||
MapValue(value)
|
||||
};
|
||||
}
|
||||
return Array.Empty<MetricsSetValue>();
|
||||
}
|
||||
|
||||
private MetricsSetValue[] MapMultipleValues(object[][] values)
|
||||
{
|
||||
if (values != null && values.Length > 0)
|
||||
{
|
||||
return values.Select(v => MapValue(v)).ToArray();
|
||||
}
|
||||
return Array.Empty<MetricsSetValue>();
|
||||
}
|
||||
|
||||
private MetricsSetValue MapValue(object[] value)
|
||||
{
|
||||
if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string].");
|
||||
|
||||
return new MetricsSetValue
|
||||
{
|
||||
Timestamp = ToTimestamp(value[0]),
|
||||
Value = ToValue(value[1])
|
||||
};
|
||||
}
|
||||
|
||||
private string GetInstanceNameForNode(RunningContainer node)
|
||||
{
|
||||
var ip = node.Pod.Ip;
|
||||
var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
|
||||
return $"{ip}:{port}";
|
||||
}
|
||||
|
||||
private string GetInstanceStringForNode(RunningContainer node)
|
||||
{
|
||||
return "{instance=\"" + GetInstanceNameForNode(node) + "\"}";
|
||||
}
|
||||
|
||||
private string GetQueryTimeRange()
|
||||
{
|
||||
return "[12h]";
|
||||
}
|
||||
|
||||
private double ToValue(object v)
|
||||
{
|
||||
return Convert.ToDouble(v, CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private DateTime ToTimestamp(object v)
|
||||
{
|
||||
var unixSeconds = ToValue(v);
|
||||
return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
|
||||
}
|
||||
}
|
||||
|
||||
public class Metrics
|
||||
{
|
||||
public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>();
|
||||
}
|
||||
|
||||
public class MetricsSet
|
||||
{
|
||||
public string Name { get; set; } = string.Empty;
|
||||
public string Instance { get; set; } = string.Empty;
|
||||
public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
|
||||
}
|
||||
|
||||
public class MetricsSetValue
|
||||
{
|
||||
public DateTime Timestamp { get; set; }
|
||||
public double Value { get; set; }
|
||||
}
|
||||
|
||||
public class PrometheusQueryResponse
|
||||
{
|
||||
public string status { get; set; } = string.Empty;
|
||||
public PrometheusQueryResponseData data { get; set; } = new();
|
||||
}
|
||||
|
||||
public class PrometheusQueryResponseData
|
||||
{
|
||||
public string resultType { get; set; } = string.Empty;
|
||||
public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty<PrometheusQueryResponseDataResultEntry>();
|
||||
}
|
||||
|
||||
public class PrometheusQueryResponseDataResultEntry
|
||||
{
|
||||
public ResultEntryMetric metric { get; set; } = new();
|
||||
public object[] value { get; set; } = Array.Empty<object>();
|
||||
public object[][] values { get; set; } = Array.Empty<object[]>();
|
||||
}
|
||||
|
||||
public class ResultEntryMetric
|
||||
{
|
||||
public string __name__ { get; set; } = string.Empty;
|
||||
public string instance { get; set; } = string.Empty;
|
||||
public string job { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
public class PrometheusAllNamesResponse
|
||||
{
|
||||
public string status { get; set; } = string.Empty;
|
||||
public string[] data { get; set; } = Array.Empty<string>();
|
||||
}
|
||||
}
|
|
@ -0,0 +1,17 @@
|
|||
using KubernetesWorkflow;
|
||||
|
||||
namespace DistTestCore.Metrics
|
||||
{
|
||||
public class PrometheusContainerRecipe : ContainerRecipeFactory
|
||||
{
|
||||
protected override string Image => "thatbenbierens/prometheus-envconf:latest";
|
||||
|
||||
protected override void Initialize(StartupConfig startupConfig)
|
||||
{
|
||||
var config = startupConfig.Get<PrometheusStartupConfig>();
|
||||
|
||||
AddExposedPortAndVar("PROM_PORT");
|
||||
AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,12 @@
|
|||
namespace DistTestCore.Metrics
|
||||
{
|
||||
public class PrometheusStartupConfig
|
||||
{
|
||||
public PrometheusStartupConfig(string prometheusConfigBase64)
|
||||
{
|
||||
PrometheusConfigBase64 = prometheusConfigBase64;
|
||||
}
|
||||
|
||||
public string PrometheusConfigBase64 { get; }
|
||||
}
|
||||
}
|
|
@ -1,5 +1,6 @@
|
|||
using DistTestCore.Codex;
|
||||
using DistTestCore.CodexLogsAndMetrics;
|
||||
using DistTestCore.Metrics;
|
||||
using NUnit.Framework;
|
||||
|
||||
namespace DistTestCore
|
||||
|
@ -11,7 +12,7 @@ namespace DistTestCore
|
|||
TestFile? DownloadContent(ContentId contentId);
|
||||
void ConnectToPeer(IOnlineCodexNode node);
|
||||
ICodexNodeLog DownloadLog();
|
||||
//IMetricsAccess Metrics { get; }
|
||||
IMetricsAccess Metrics { get; }
|
||||
//IMarketplaceAccess Marketplace { get; }
|
||||
}
|
||||
|
||||
|
@ -21,15 +22,17 @@ namespace DistTestCore
|
|||
private const string UploadFailedMessage = "Unable to store block";
|
||||
private readonly TestLifecycle lifecycle;
|
||||
|
||||
public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group)
|
||||
public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess)
|
||||
{
|
||||
this.lifecycle = lifecycle;
|
||||
CodexAccess = codexAccess;
|
||||
Group = group;
|
||||
Metrics = metricsAccess;
|
||||
}
|
||||
|
||||
public CodexAccess CodexAccess { get; }
|
||||
public CodexNodeGroup Group { get; }
|
||||
public IMetricsAccess Metrics { get; }
|
||||
|
||||
public string GetName()
|
||||
{
|
||||
|
|
|
@ -0,0 +1,65 @@
|
|||
using DistTestCore.Codex;
|
||||
using DistTestCore.Metrics;
|
||||
using KubernetesWorkflow;
|
||||
using System.Text;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public class PrometheusStarter
|
||||
{
|
||||
private readonly TestLifecycle lifecycle;
|
||||
private readonly WorkflowCreator workflowCreator;
|
||||
|
||||
public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
|
||||
{
|
||||
this.lifecycle = lifecycle;
|
||||
this.workflowCreator = workflowCreator;
|
||||
}
|
||||
|
||||
public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers)
|
||||
{
|
||||
if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
|
||||
|
||||
Log($"Starting metrics server for {containers.Describe()}");
|
||||
var startupConfig = new StartupConfig();
|
||||
startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers)));
|
||||
|
||||
var workflow = workflowCreator.CreateWorkflow();
|
||||
var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
|
||||
if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
|
||||
|
||||
Log("Metrics server started.");
|
||||
|
||||
return new CodexNodeMetricsAccessFactory(runningContainers);
|
||||
}
|
||||
|
||||
private string GeneratePrometheusConfig(RunningContainer[] nodes)
|
||||
{
|
||||
var config = "";
|
||||
config += "global:\n";
|
||||
config += " scrape_interval: 30s\n";
|
||||
config += " scrape_timeout: 10s\n";
|
||||
config += "\n";
|
||||
config += "scrape_configs:\n";
|
||||
config += " - job_name: services\n";
|
||||
config += " metrics_path: /metrics\n";
|
||||
config += " static_configs:\n";
|
||||
config += " - targets:\n";
|
||||
|
||||
foreach (var node in nodes)
|
||||
{
|
||||
var ip = node.Pod.Ip;
|
||||
var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
|
||||
config += $" - '{ip}:{port}'\n";
|
||||
}
|
||||
|
||||
var bytes = Encoding.ASCII.GetBytes(config);
|
||||
return Convert.ToBase64String(bytes);
|
||||
}
|
||||
|
||||
private void Log(string msg)
|
||||
{
|
||||
lifecycle.Log.Log(msg);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,20 +1,27 @@
|
|||
using DistTestCore.CodexLogsAndMetrics;
|
||||
using KubernetesWorkflow;
|
||||
using Logging;
|
||||
|
||||
namespace DistTestCore
|
||||
{
|
||||
public class TestLifecycle
|
||||
{
|
||||
private readonly WorkflowCreator workflowCreator;
|
||||
|
||||
public TestLifecycle(Configuration configuration)
|
||||
{
|
||||
Log = new TestLog(configuration.GetLogConfig());
|
||||
workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration());
|
||||
|
||||
FileManager = new FileManager(Log, configuration);
|
||||
CodexStarter = new CodexStarter(this, configuration);
|
||||
CodexStarter = new CodexStarter(this, workflowCreator);
|
||||
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
|
||||
}
|
||||
|
||||
public TestLog Log { get; }
|
||||
public FileManager FileManager { get; }
|
||||
public CodexStarter CodexStarter { get; }
|
||||
public PrometheusStarter PrometheusStarter { get; }
|
||||
|
||||
public void DeleteAllResources()
|
||||
{
|
||||
|
|
|
@ -17,16 +17,23 @@
|
|||
public Port[] ExposedPorts { get; }
|
||||
public Port[] InternalPorts { get; }
|
||||
public EnvVar[] EnvVars { get; }
|
||||
|
||||
public Port GetPortByTag(string tag)
|
||||
{
|
||||
return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag);
|
||||
}
|
||||
}
|
||||
|
||||
public class Port
|
||||
{
|
||||
public Port(int number)
|
||||
public Port(int number, string tag)
|
||||
{
|
||||
Number = number;
|
||||
Tag = tag;
|
||||
}
|
||||
|
||||
public int Number { get; }
|
||||
public string Tag { get; }
|
||||
}
|
||||
|
||||
public class EnvVar
|
||||
|
|
|
@ -28,28 +28,28 @@
|
|||
protected int ContainerNumber { get; private set; } = 0;
|
||||
protected abstract void Initialize(StartupConfig config);
|
||||
|
||||
protected Port AddExposedPort()
|
||||
protected Port AddExposedPort(string tag = "")
|
||||
{
|
||||
var p = factory.CreatePort();
|
||||
var p = factory.CreatePort(tag);
|
||||
exposedPorts.Add(p);
|
||||
return p;
|
||||
}
|
||||
|
||||
protected Port AddInternalPort()
|
||||
protected Port AddInternalPort(string tag = "")
|
||||
{
|
||||
var p = factory.CreatePort();
|
||||
var p = factory.CreatePort(tag);
|
||||
internalPorts.Add(p);
|
||||
return p;
|
||||
}
|
||||
|
||||
protected void AddExposedPortAndVar(string name)
|
||||
protected void AddExposedPortAndVar(string name, string tag = "")
|
||||
{
|
||||
AddEnvVar(name, AddExposedPort());
|
||||
AddEnvVar(name, AddExposedPort(tag));
|
||||
}
|
||||
|
||||
protected void AddInternalPortAndVar(string name)
|
||||
protected void AddInternalPortAndVar(string name, string tag = "")
|
||||
{
|
||||
AddEnvVar(name, AddInternalPort());
|
||||
AddEnvVar(name, AddInternalPort(tag));
|
||||
}
|
||||
|
||||
protected void AddEnvVar(string name, string value)
|
||||
|
|
|
@ -277,7 +277,7 @@ namespace KubernetesWorkflow
|
|||
foreach (var port in recipe.ExposedPorts)
|
||||
{
|
||||
var servicePort = workflowNumberSource.GetServicePort();
|
||||
usedPorts.Add(new Port(servicePort));
|
||||
usedPorts.Add(new Port(servicePort, ""));
|
||||
|
||||
result.Add(new V1ServicePort
|
||||
{
|
||||
|
|
|
@ -7,9 +7,9 @@ namespace KubernetesWorkflow
|
|||
{
|
||||
private NumberSource portNumberSource = new NumberSource(8080);
|
||||
|
||||
public Port CreatePort()
|
||||
public Port CreatePort(string tag)
|
||||
{
|
||||
return new Port(portNumberSource.GetNextNumber());
|
||||
return new Port(portNumberSource.GetNextNumber(), tag);
|
||||
}
|
||||
|
||||
public EnvVar CreateEnvVar(string name, int value)
|
||||
|
|
|
@ -12,6 +12,11 @@
|
|||
public StartupConfig StartupConfig { get; }
|
||||
public RunningPod RunningPod { get; }
|
||||
public RunningContainer[] Containers { get; }
|
||||
|
||||
public string Describe()
|
||||
{
|
||||
return $"[{RunningPod.Ip}]";
|
||||
}
|
||||
}
|
||||
|
||||
public class RunningContainer
|
||||
|
|
|
@ -78,30 +78,30 @@ namespace Tests.BasicTests
|
|||
log.AssertLogContains("Uploaded file");
|
||||
}
|
||||
|
||||
//[Test]
|
||||
//public void TwoMetricsExample()
|
||||
//{
|
||||
// var group = SetupCodexNodes(2)
|
||||
// .EnableMetrics()
|
||||
// .BringOnline();
|
||||
[Test]
|
||||
public void TwoMetricsExample()
|
||||
{
|
||||
var group = SetupCodexNodes(2)
|
||||
.EnableMetrics()
|
||||
.BringOnline();
|
||||
|
||||
// var group2 = SetupCodexNodes(2)
|
||||
// .EnableMetrics()
|
||||
// .BringOnline();
|
||||
var group2 = SetupCodexNodes(2)
|
||||
.EnableMetrics()
|
||||
.BringOnline();
|
||||
|
||||
// var primary = group[0];
|
||||
// var secondary = group[1];
|
||||
// var primary2 = group2[0];
|
||||
// var secondary2 = group2[1];
|
||||
var primary = group[0];
|
||||
var secondary = group[1];
|
||||
var primary2 = group2[0];
|
||||
var secondary2 = group2[1];
|
||||
|
||||
// primary.ConnectToPeer(secondary);
|
||||
// primary2.ConnectToPeer(secondary2);
|
||||
primary.ConnectToPeer(secondary);
|
||||
primary2.ConnectToPeer(secondary2);
|
||||
|
||||
// Thread.Sleep(TimeSpan.FromMinutes(5));
|
||||
Thread.Sleep(TimeSpan.FromMinutes(5));
|
||||
|
||||
// primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
|
||||
// primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
|
||||
//}
|
||||
primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
|
||||
primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1));
|
||||
}
|
||||
|
||||
//[Test]
|
||||
//public void MarketplaceExample()
|
||||
|
|
Loading…
Reference in New Issue