diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 9c7b87d..e49dd84 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -4,6 +4,8 @@ namespace DistTestCore.Codex { public class CodexContainerRecipe : ContainerRecipeFactory { + public const string MetricsPortTag = "metrics_port"; + protected override string Image => "thatbenbierens/nim-codex:sha-b204837"; protected override void Initialize(StartupConfig startupConfig) @@ -28,7 +30,7 @@ namespace DistTestCore.Codex if (config.MetricsEnabled) { AddEnvVar("METRICS_ADDR", "0.0.0.0"); - AddInternalPortAndVar("METRICS_PORT"); + AddInternalPortAndVar("METRICS_PORT", tag: MetricsPortTag); } } } diff --git a/DistTestCore/CodexNodeFactory.cs b/DistTestCore/CodexNodeFactory.cs new file mode 100644 index 0000000..3dce8aa --- /dev/null +++ b/DistTestCore/CodexNodeFactory.cs @@ -0,0 +1,28 @@ +using DistTestCore.Codex; +using DistTestCore.Metrics; + +namespace DistTestCore +{ + public interface ICodexNodeFactory + { + OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group); + } + + public class CodexNodeFactory : ICodexNodeFactory + { + private readonly TestLifecycle lifecycle; + private readonly IMetricsAccessFactory metricsAccessFactory; + + public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory) + { + this.lifecycle = lifecycle; + this.metricsAccessFactory = metricsAccessFactory; + } + + public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group) + { + var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container); + return new OnlineCodexNode(lifecycle, access, group, metricsAccess); + } + } +} diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 2c5c312..da4313f 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -14,12 +14,12 @@ namespace DistTestCore { private readonly TestLifecycle lifecycle; - public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers) + public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory) { this.lifecycle = lifecycle; Setup = setup; Containers = containers; - Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c)).ToArray(); + Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray(); } public IOnlineCodexNode this[int index] @@ -73,14 +73,13 @@ namespace DistTestCore public string Describe() { - var orderNumber = Containers.RunningPod.Ip; - return $"CodexNodeGroup@{orderNumber}-{Setup.Describe()}"; + return $"CodexNodeGroup@{Containers.Describe()}-{Setup.Describe()}"; } - private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c) + private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) { var access = new CodexAccess(c); - return new OnlineCodexNode(lifecycle, access, this); + return factory.CreateOnlineCodexNode(access, this); } } } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index d7bc0ab..65c9a32 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -5,31 +5,27 @@ namespace DistTestCore { public class CodexStarter { - private readonly WorkflowCreator workflowCreator; private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; - public CodexStarter(TestLifecycle lifecycle, Configuration configuration) + public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) { - workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; } public List RunningGroups { get; } = new List(); public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { - Log($"Starting {codexSetup.Describe()}..."); + var containers = StartCodexContainers(codexSetup); - var workflow = CreateWorkflow(); - var startupConfig = new StartupConfig(); - startupConfig.Add(codexSetup); + var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers); - var runningContainers = workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig); + var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory); - var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers); - RunningGroups.Add(group); + var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); - Log($"Started at '{group.Containers.RunningPod.Ip}'"); return group; } @@ -55,6 +51,26 @@ namespace DistTestCore var workflow = CreateWorkflow(); workflow.DownloadContainerLog(container, logHandler); } + + private RunningContainers StartCodexContainers(CodexSetup codexSetup) + { + Log($"Starting {codexSetup.Describe()}..."); + + var workflow = CreateWorkflow(); + var startupConfig = new StartupConfig(); + startupConfig.Add(codexSetup); + + return workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig); + } + + private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory) + { + var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); + RunningGroups.Add(group); + + Log($"Started at '{group.Containers.RunningPod.Ip}'"); + return group; + } private StartupWorkflow CreateWorkflow() { diff --git a/DistTestCore/Metrics/MetricsAccess.cs b/DistTestCore/Metrics/MetricsAccess.cs new file mode 100644 index 0000000..3287ea7 --- /dev/null +++ b/DistTestCore/Metrics/MetricsAccess.cs @@ -0,0 +1,65 @@ +using KubernetesWorkflow; +using NUnit.Framework; +using NUnit.Framework.Constraints; +using Utils; + +namespace DistTestCore.Metrics +{ + public interface IMetricsAccess + { + void AssertThat(string metricName, IResolveConstraint constraint, string message = ""); + } + + public class MetricsUnavailable : IMetricsAccess + { + public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") + { + Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it."); + throw new InvalidOperationException(); + } + } + + public class MetricsAccess : IMetricsAccess + { + private readonly MetricsQuery query; + private readonly RunningContainer node; + + public MetricsAccess(MetricsQuery query, RunningContainer node) + { + this.query = query; + this.node = node; + } + + public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") + { + var metricSet = GetMetricWithTimeout(metricName); + var metricValue = metricSet.Values[0].Value; + Assert.That(metricValue, constraint, message); + } + + private MetricsSet GetMetricWithTimeout(string metricName) + { + var start = DateTime.UtcNow; + + while (true) + { + var mostRecent = GetMostRecent(metricName); + if (mostRecent != null) return mostRecent; + if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout()) + { + Assert.Fail($"Timeout: Unable to get metric '{metricName}'."); + throw new TimeoutException(); + } + + Time.Sleep(TimeSpan.FromSeconds(2)); + } + } + + private MetricsSet? GetMostRecent(string metricName) + { + var result = query.GetMostRecent(metricName, node); + if (result == null) return null; + return result.Sets.LastOrDefault(); + } + } +} diff --git a/DistTestCore/Metrics/MetricsAccessFactory.cs b/DistTestCore/Metrics/MetricsAccessFactory.cs new file mode 100644 index 0000000..fcf5dfb --- /dev/null +++ b/DistTestCore/Metrics/MetricsAccessFactory.cs @@ -0,0 +1,33 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Metrics +{ + public interface IMetricsAccessFactory + { + IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer); + } + + public class MetricsUnavailableAccessFactory : IMetricsAccessFactory + { + public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) + { + return new MetricsUnavailable(); + } + } + + public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory + { + private readonly RunningContainers prometheusContainer; + + public CodexNodeMetricsAccessFactory(RunningContainers prometheusContainer) + { + this.prometheusContainer = prometheusContainer; + } + + public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) + { + var query = new MetricsQuery(prometheusContainer); + return new MetricsAccess(query, codexContainer); + } + } +} diff --git a/DistTestCore/Metrics/MetricsDownloader.cs b/DistTestCore/Metrics/MetricsDownloader.cs new file mode 100644 index 0000000..3d79752 --- /dev/null +++ b/DistTestCore/Metrics/MetricsDownloader.cs @@ -0,0 +1,98 @@ +using Logging; +using System.Globalization; + +namespace DistTestCore.Metrics +{ + public class MetricsDownloader + { + private readonly TestLog log; + private readonly Dictionary activePrometheuses; + + public MetricsDownloader(TestLog log, Dictionary activePrometheuses) + { + this.log = log; + this.activePrometheuses = activePrometheuses; + } + + public void DownloadAllMetrics() + { + foreach (var pair in activePrometheuses) + { + DownloadAllMetrics(pair.Key, pair.Value); + } + } + + private void DownloadAllMetrics(MetricsQuery query, OnlineCodexNode[] nodes) + { + foreach (var node in nodes) + { + DownloadAllMetricsForNode(query, node); + } + } + + private void DownloadAllMetricsForNode(MetricsQuery query, OnlineCodexNode node) + { + var metrics = query.GetAllMetricsForNode(node.CodexAccess.Container); + if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return; + + var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray(); + var map = CreateValueMap(metrics); + + WriteToFile(node.GetName(), headers, map); + } + + private void WriteToFile(string nodeName, string[] headers, Dictionary> map) + { + var file = log.CreateSubfile("csv"); + log.Log($"Downloading metrics for {nodeName} to file {file.FilenameWithoutPath}"); + + file.WriteRaw(string.Join(",", headers)); + + foreach (var pair in map) + { + file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value))); + } + } + + private Dictionary> CreateValueMap(Metrics metrics) + { + var map = CreateForAllTimestamps(metrics); + foreach (var metric in metrics.Sets) + { + AddToMap(map, metric); + } + return map; + + } + + private Dictionary> CreateForAllTimestamps(Metrics metrics) + { + var result = new Dictionary>(); + var timestamps = metrics.Sets.SelectMany(s => s.Values).Select(v => v.Timestamp).Distinct().ToArray(); + foreach (var timestamp in timestamps) result.Add(timestamp, new List()); + return result; + } + + private void AddToMap(Dictionary> map, MetricsSet metric) + { + foreach (var key in map.Keys) + { + map[key].Add(GetValueAtTimestamp(key, metric)); + } + } + + private string GetValueAtTimestamp(DateTime key, MetricsSet metric) + { + var value = metric.Values.SingleOrDefault(v => v.Timestamp == key); + if (value == null) return ""; + return value.Value.ToString(CultureInfo.InvariantCulture); + } + + private string FormatTimestamp(DateTime key) + { + var origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + var diff = key - origin; + return Math.Floor(diff.TotalSeconds).ToString(CultureInfo.InvariantCulture); + } + } +} diff --git a/DistTestCore/Metrics/MetricsQuery.cs b/DistTestCore/Metrics/MetricsQuery.cs new file mode 100644 index 0000000..c06fc8d --- /dev/null +++ b/DistTestCore/Metrics/MetricsQuery.cs @@ -0,0 +1,195 @@ +using DistTestCore.Codex; +using KubernetesWorkflow; +using System.Globalization; + +namespace DistTestCore.Metrics +{ + public class MetricsQuery + { + private readonly Http http; + + public MetricsQuery(RunningContainers runningContainers) + { + RunningContainers = runningContainers; + + http = new Http( + runningContainers.RunningPod.Cluster.IP, + runningContainers.Containers[0].ServicePorts[0].Number, + "api/v1"); + } + + public RunningContainers RunningContainers { get; } + + public Metrics? GetMostRecent(string metricName, RunningContainer node) + { + var response = GetLastOverTime(metricName, GetInstanceStringForNode(node)); + if (response == null) return null; + + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Instance = r.metric.instance, + Values = MapSingleValue(r.value) + }; + }).ToArray() + }; + } + + public Metrics? GetMetrics(string metricName) + { + var response = GetAll(metricName); + if (response == null) return null; + return MapResponseToMetrics(response); + } + + public Metrics? GetAllMetricsForNode(RunningContainer node) + { + var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return MapResponseToMetrics(response); + } + + private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) + { + var response = http.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); + if (response.status != "success") return null; + return response; + } + + private PrometheusQueryResponse? GetAll(string metricName) + { + var response = http.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}"); + if (response.status != "success") return null; + return response; + } + + private Metrics MapResponseToMetrics(PrometheusQueryResponse response) + { + return new Metrics + { + Sets = response.data.result.Select(r => + { + return new MetricsSet + { + Name = r.metric.__name__, + Instance = r.metric.instance, + Values = MapMultipleValues(r.values) + }; + }).ToArray() + }; + } + + private MetricsSetValue[] MapSingleValue(object[] value) + { + if (value != null && value.Length > 0) + { + return new[] + { + MapValue(value) + }; + } + return Array.Empty(); + } + + private MetricsSetValue[] MapMultipleValues(object[][] values) + { + if (values != null && values.Length > 0) + { + return values.Select(v => MapValue(v)).ToArray(); + } + return Array.Empty(); + } + + private MetricsSetValue MapValue(object[] value) + { + if (value.Length != 2) throw new InvalidOperationException("Expected value to be [double, string]."); + + return new MetricsSetValue + { + Timestamp = ToTimestamp(value[0]), + Value = ToValue(value[1]) + }; + } + + private string GetInstanceNameForNode(RunningContainer node) + { + var ip = node.Pod.Ip; + var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; + return $"{ip}:{port}"; + } + + private string GetInstanceStringForNode(RunningContainer node) + { + return "{instance=\"" + GetInstanceNameForNode(node) + "\"}"; + } + + private string GetQueryTimeRange() + { + return "[12h]"; + } + + private double ToValue(object v) + { + return Convert.ToDouble(v, CultureInfo.InvariantCulture); + } + + private DateTime ToTimestamp(object v) + { + var unixSeconds = ToValue(v); + return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds); + } + } + + public class Metrics + { + public MetricsSet[] Sets { get; set; } = Array.Empty(); + } + + public class MetricsSet + { + public string Name { get; set; } = string.Empty; + public string Instance { get; set; } = string.Empty; + public MetricsSetValue[] Values { get; set; } = Array.Empty(); + } + + public class MetricsSetValue + { + public DateTime Timestamp { get; set; } + public double Value { get; set; } + } + + public class PrometheusQueryResponse + { + public string status { get; set; } = string.Empty; + public PrometheusQueryResponseData data { get; set; } = new(); + } + + public class PrometheusQueryResponseData + { + public string resultType { get; set; } = string.Empty; + public PrometheusQueryResponseDataResultEntry[] result { get; set; } = Array.Empty(); + } + + public class PrometheusQueryResponseDataResultEntry + { + public ResultEntryMetric metric { get; set; } = new(); + public object[] value { get; set; } = Array.Empty(); + public object[][] values { get; set; } = Array.Empty(); + } + + public class ResultEntryMetric + { + public string __name__ { get; set; } = string.Empty; + public string instance { get; set; } = string.Empty; + public string job { get; set; } = string.Empty; + } + + public class PrometheusAllNamesResponse + { + public string status { get; set; } = string.Empty; + public string[] data { get; set; } = Array.Empty(); + } +} diff --git a/DistTestCore/Metrics/PrometheusContainerRecipe.cs b/DistTestCore/Metrics/PrometheusContainerRecipe.cs new file mode 100644 index 0000000..5152ff7 --- /dev/null +++ b/DistTestCore/Metrics/PrometheusContainerRecipe.cs @@ -0,0 +1,17 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Metrics +{ + public class PrometheusContainerRecipe : ContainerRecipeFactory + { + protected override string Image => "thatbenbierens/prometheus-envconf:latest"; + + protected override void Initialize(StartupConfig startupConfig) + { + var config = startupConfig.Get(); + + AddExposedPortAndVar("PROM_PORT"); + AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64); + } + } +} diff --git a/DistTestCore/Metrics/PrometheusStartupConfig.cs b/DistTestCore/Metrics/PrometheusStartupConfig.cs new file mode 100644 index 0000000..7bf7fe6 --- /dev/null +++ b/DistTestCore/Metrics/PrometheusStartupConfig.cs @@ -0,0 +1,12 @@ +namespace DistTestCore.Metrics +{ + public class PrometheusStartupConfig + { + public PrometheusStartupConfig(string prometheusConfigBase64) + { + PrometheusConfigBase64 = prometheusConfigBase64; + } + + public string PrometheusConfigBase64 { get; } + } +} diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 8f85f91..0efa51a 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Metrics; using NUnit.Framework; namespace DistTestCore @@ -11,7 +12,7 @@ namespace DistTestCore TestFile? DownloadContent(ContentId contentId); void ConnectToPeer(IOnlineCodexNode node); ICodexNodeLog DownloadLog(); - //IMetricsAccess Metrics { get; } + IMetricsAccess Metrics { get; } //IMarketplaceAccess Marketplace { get; } } @@ -21,15 +22,17 @@ namespace DistTestCore private const string UploadFailedMessage = "Unable to store block"; private readonly TestLifecycle lifecycle; - public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group) + public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess) { this.lifecycle = lifecycle; CodexAccess = codexAccess; Group = group; + Metrics = metricsAccess; } public CodexAccess CodexAccess { get; } public CodexNodeGroup Group { get; } + public IMetricsAccess Metrics { get; } public string GetName() { diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs new file mode 100644 index 0000000..a58b3e4 --- /dev/null +++ b/DistTestCore/PrometheusStarter.cs @@ -0,0 +1,65 @@ +using DistTestCore.Codex; +using DistTestCore.Metrics; +using KubernetesWorkflow; +using System.Text; + +namespace DistTestCore +{ + public class PrometheusStarter + { + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + + public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + } + + public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers) + { + if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); + + Log($"Starting metrics server for {containers.Describe()}"); + var startupConfig = new StartupConfig(); + startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers))); + + var workflow = workflowCreator.CreateWorkflow(); + var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); + if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); + + Log("Metrics server started."); + + return new CodexNodeMetricsAccessFactory(runningContainers); + } + + private string GeneratePrometheusConfig(RunningContainer[] nodes) + { + var config = ""; + config += "global:\n"; + config += " scrape_interval: 30s\n"; + config += " scrape_timeout: 10s\n"; + config += "\n"; + config += "scrape_configs:\n"; + config += " - job_name: services\n"; + config += " metrics_path: /metrics\n"; + config += " static_configs:\n"; + config += " - targets:\n"; + + foreach (var node in nodes) + { + var ip = node.Pod.Ip; + var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; + config += $" - '{ip}:{port}'\n"; + } + + var bytes = Encoding.ASCII.GetBytes(config); + return Convert.ToBase64String(bytes); + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 1267216..a5127a7 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,20 +1,27 @@ using DistTestCore.CodexLogsAndMetrics; +using KubernetesWorkflow; using Logging; namespace DistTestCore { public class TestLifecycle { + private readonly WorkflowCreator workflowCreator; + public TestLifecycle(Configuration configuration) { Log = new TestLog(configuration.GetLogConfig()); + workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); + FileManager = new FileManager(Log, configuration); - CodexStarter = new CodexStarter(this, configuration); + CodexStarter = new CodexStarter(this, workflowCreator); + PrometheusStarter = new PrometheusStarter(this, workflowCreator); } public TestLog Log { get; } public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } + public PrometheusStarter PrometheusStarter { get; } public void DeleteAllResources() { diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index 7e8d90e..f676c7f 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -17,16 +17,23 @@ public Port[] ExposedPorts { get; } public Port[] InternalPorts { get; } public EnvVar[] EnvVars { get; } + + public Port GetPortByTag(string tag) + { + return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag); + } } public class Port { - public Port(int number) + public Port(int number, string tag) { Number = number; + Tag = tag; } public int Number { get; } + public string Tag { get; } } public class EnvVar diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 60ea91e..6c6a3ee 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -28,28 +28,28 @@ protected int ContainerNumber { get; private set; } = 0; protected abstract void Initialize(StartupConfig config); - protected Port AddExposedPort() + protected Port AddExposedPort(string tag = "") { - var p = factory.CreatePort(); + var p = factory.CreatePort(tag); exposedPorts.Add(p); return p; } - protected Port AddInternalPort() + protected Port AddInternalPort(string tag = "") { - var p = factory.CreatePort(); + var p = factory.CreatePort(tag); internalPorts.Add(p); return p; } - protected void AddExposedPortAndVar(string name) + protected void AddExposedPortAndVar(string name, string tag = "") { - AddEnvVar(name, AddExposedPort()); + AddEnvVar(name, AddExposedPort(tag)); } - protected void AddInternalPortAndVar(string name) + protected void AddInternalPortAndVar(string name, string tag = "") { - AddEnvVar(name, AddInternalPort()); + AddEnvVar(name, AddInternalPort(tag)); } protected void AddEnvVar(string name, string value) diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index eb492a9..394f86e 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -277,7 +277,7 @@ namespace KubernetesWorkflow foreach (var port in recipe.ExposedPorts) { var servicePort = workflowNumberSource.GetServicePort(); - usedPorts.Add(new Port(servicePort)); + usedPorts.Add(new Port(servicePort, "")); result.Add(new V1ServicePort { diff --git a/KubernetesWorkflow/RecipeComponentFactory.cs b/KubernetesWorkflow/RecipeComponentFactory.cs index f99f345..cf1f67b 100644 --- a/KubernetesWorkflow/RecipeComponentFactory.cs +++ b/KubernetesWorkflow/RecipeComponentFactory.cs @@ -7,9 +7,9 @@ namespace KubernetesWorkflow { private NumberSource portNumberSource = new NumberSource(8080); - public Port CreatePort() + public Port CreatePort(string tag) { - return new Port(portNumberSource.GetNextNumber()); + return new Port(portNumberSource.GetNextNumber(), tag); } public EnvVar CreateEnvVar(string name, int value) diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs index 49fc65f..783c6f8 100644 --- a/KubernetesWorkflow/RunningContainers.cs +++ b/KubernetesWorkflow/RunningContainers.cs @@ -12,6 +12,11 @@ public StartupConfig StartupConfig { get; } public RunningPod RunningPod { get; } public RunningContainer[] Containers { get; } + + public string Describe() + { + return $"[{RunningPod.Ip}]"; + } } public class RunningContainer diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 6f6ce14..bf0c331 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -78,30 +78,30 @@ namespace Tests.BasicTests log.AssertLogContains("Uploaded file"); } - //[Test] - //public void TwoMetricsExample() - //{ - // var group = SetupCodexNodes(2) - // .EnableMetrics() - // .BringOnline(); + [Test] + public void TwoMetricsExample() + { + var group = SetupCodexNodes(2) + .EnableMetrics() + .BringOnline(); - // var group2 = SetupCodexNodes(2) - // .EnableMetrics() - // .BringOnline(); + var group2 = SetupCodexNodes(2) + .EnableMetrics() + .BringOnline(); - // var primary = group[0]; - // var secondary = group[1]; - // var primary2 = group2[0]; - // var secondary2 = group2[1]; + var primary = group[0]; + var secondary = group[1]; + var primary2 = group2[0]; + var secondary2 = group2[1]; - // primary.ConnectToPeer(secondary); - // primary2.ConnectToPeer(secondary2); + primary.ConnectToPeer(secondary); + primary2.ConnectToPeer(secondary2); - // Thread.Sleep(TimeSpan.FromMinutes(5)); + Thread.Sleep(TimeSpan.FromMinutes(5)); - // primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); - // primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); - //} + primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + } //[Test] //public void MarketplaceExample()