diff --git a/CodexPlugin/CodexContainerRecipe.cs b/CodexPlugin/CodexContainerRecipe.cs index 1eb72f7..08f5eae 100644 --- a/CodexPlugin/CodexContainerRecipe.cs +++ b/CodexPlugin/CodexContainerRecipe.cs @@ -67,15 +67,15 @@ namespace CodexPlugin { AddEnvVar("CODEX_BLOCK_MN", config.BlockMaintenanceNumber.ToString()!); } - //if (config.MetricsMode != Metrics.MetricsMode.None) - //{ - // var metricsPort = AddInternalPort(MetricsPortTag); - // AddEnvVar("CODEX_METRICS", "true"); - // AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0"); - // AddEnvVar("CODEX_METRICS_PORT", metricsPort); - // AddPodAnnotation("prometheus.io/scrape", "true"); - // AddPodAnnotation("prometheus.io/port", metricsPort.Number.ToString()); - //} + if (config.MetricsEnabled) + { + var metricsPort = AddInternalPort(MetricsPortTag); + AddEnvVar("CODEX_METRICS", "true"); + AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0"); + AddEnvVar("CODEX_METRICS_PORT", metricsPort); + AddPodAnnotation("prometheus.io/scrape", "true"); + AddPodAnnotation("prometheus.io/port", metricsPort.Number.ToString()); + } //if (config.MarketplaceConfig != null) //{ diff --git a/CodexPlugin/CodexNodeGroup.cs b/CodexPlugin/CodexNodeGroup.cs index c2e619e..e3522dd 100644 --- a/CodexPlugin/CodexNodeGroup.cs +++ b/CodexPlugin/CodexNodeGroup.cs @@ -1,10 +1,11 @@ using Core; using KubernetesWorkflow; +using MetricsPlugin; using System.Collections; namespace CodexPlugin { - public interface ICodexNodeGroup : IEnumerable + public interface ICodexNodeGroup : IEnumerable, IManyMetricScrapeTargets { void BringOffline(); IOnlineCodexNode this[int index] { get; } @@ -41,6 +42,7 @@ namespace CodexPlugin public RunningContainers[] Containers { get; private set; } public OnlineCodexNode[] Nodes { get; private set; } public CodexDebugVersionResponse Version { get; private set; } + public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray(); public IEnumerator GetEnumerator() { diff --git a/CodexPlugin/CodexPlugin.csproj b/CodexPlugin/CodexPlugin.csproj index 1f910c4..4619e26 100644 --- a/CodexPlugin/CodexPlugin.csproj +++ b/CodexPlugin/CodexPlugin.csproj @@ -13,6 +13,7 @@ + diff --git a/CodexPlugin/CodexSetup.cs b/CodexPlugin/CodexSetup.cs index 7602aca..ff777ff 100644 --- a/CodexPlugin/CodexSetup.cs +++ b/CodexPlugin/CodexSetup.cs @@ -12,7 +12,7 @@ namespace CodexPlugin ICodexSetup WithBlockTTL(TimeSpan duration); ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration); ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks); - //ICodexSetup EnableMetrics(); + ICodexSetup EnableMetrics(); //ICodexSetup EnableMarketplace(TestToken initialBalance); //ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther); //ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator); @@ -70,11 +70,11 @@ namespace CodexPlugin return this; } - //public ICodexSetup EnableMetrics() - //{ - // MetricsMode = Metrics.MetricsMode.Record; - // return this; - //} + public ICodexSetup EnableMetrics() + { + MetricsEnabled = true; + return this; + } //public ICodexSetup EnableMarketplace(TestToken initialBalance) //{ diff --git a/CodexPlugin/CodexStartupConfig.cs b/CodexPlugin/CodexStartupConfig.cs index 8563959..e2ccb1f 100644 --- a/CodexPlugin/CodexStartupConfig.cs +++ b/CodexPlugin/CodexStartupConfig.cs @@ -14,7 +14,7 @@ namespace CodexPlugin public Location Location { get; set; } public CodexLogLevel LogLevel { get; } public ByteSize? StorageQuota { get; set; } - //public MetricsMode MetricsMode { get; set; } + public bool MetricsEnabled { get; set; } //public MarketplaceInitialConfig? MarketplaceConfig { get; set; } public string? BootstrapSpr { get; set; } public int? BlockTTL { get; set; } diff --git a/CodexPlugin/OnlineCodexNode.cs b/CodexPlugin/OnlineCodexNode.cs index cbd0dcd..f3fbaa9 100644 --- a/CodexPlugin/OnlineCodexNode.cs +++ b/CodexPlugin/OnlineCodexNode.cs @@ -1,6 +1,7 @@ using Core; using FileUtils; using Logging; +using MetricsPlugin; using NUnit.Framework; using Utils; @@ -15,10 +16,9 @@ namespace CodexPlugin TrackedFile? DownloadContent(ContentId contentId, string fileLabel = ""); void ConnectToPeer(IOnlineCodexNode node); IDownloadedLog DownloadLog(int? tailLines = null); - //IMetricsAccess Metrics { get; } - //IMarketplaceAccess Marketplace { get; } CodexDebugVersionResponse Version { get; } void BringOffline(); + IMetricsScrapeTarget MetricsScrapeTarget { get; } } public class OnlineCodexNode : IOnlineCodexNode @@ -27,21 +27,26 @@ namespace CodexPlugin private const string UploadFailedMessage = "Unable to store block"; private readonly IPluginTools tools; - public OnlineCodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group/*, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess*/) + public OnlineCodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group) { this.tools = tools; CodexAccess = codexAccess; Group = group; - //Metrics = metricsAccess; - //Marketplace = marketplaceAccess; Version = new CodexDebugVersionResponse(); } public CodexAccess CodexAccess { get; } public CodexNodeGroup Group { get; } - //public IMetricsAccess Metrics { get; } - //public IMarketplaceAccess Marketplace { get; } public CodexDebugVersionResponse Version { get; private set; } + public IMetricsScrapeTarget MetricsScrapeTarget + { + get + { + var port = CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag); + if (port == null) throw new Exception("Metrics is not available for this Codex node. Please start it with the option '.EnableMetrics()' to enable it."); + return new MetricsScrapeTarget(CodexAccess.Container, port); + } + } public string GetName() { diff --git a/Core/PluginTools.cs b/Core/PluginTools.cs index f25b970..522e481 100644 --- a/Core/PluginTools.cs +++ b/Core/PluginTools.cs @@ -37,12 +37,12 @@ namespace Core private readonly IFileManager fileManager; private ILog log; - public PluginTools(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet) + public PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet) { this.log = log; + this.workflowCreator = workflowCreator; this.timeSet = timeSet; fileManager = new FileManager(log, fileManagerRootFolder); - workflowCreator = new WorkflowCreator(log, configuration); } public void ApplyLogPrefix(string prefix) diff --git a/Core/ToolsFactory.cs b/Core/ToolsFactory.cs index 581c0d3..96752b6 100644 --- a/Core/ToolsFactory.cs +++ b/Core/ToolsFactory.cs @@ -11,21 +11,21 @@ namespace Core internal class ToolsFactory : IToolsFactory { private readonly ILog log; - private readonly Configuration configuration; + private readonly WorkflowCreator workflowCreator; private readonly string fileManagerRootFolder; private readonly ITimeSet timeSet; public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet) { this.log = log; - this.configuration = configuration; + workflowCreator = new WorkflowCreator(log, configuration); this.fileManagerRootFolder = fileManagerRootFolder; this.timeSet = timeSet; } public PluginTools CreateTools() { - return new PluginTools(log, configuration, fileManagerRootFolder, timeSet); + return new PluginTools(log, workflowCreator, fileManagerRootFolder, timeSet); } } } diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index 51ea2e1..318d3ce 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -28,9 +28,9 @@ public VolumeMount[] Volumes { get; } public object[] Additionals { get; } - public Port GetPortByTag(string tag) + public Port? GetPortByTag(string tag) { - return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag); + return ExposedPorts.Concat(InternalPorts).SingleOrDefault(p => p.Tag == tag); } public override string ToString() diff --git a/MetricsPlugin/CoreInterfaceExtensions.cs b/MetricsPlugin/CoreInterfaceExtensions.cs index 0346857..03ea973 100644 --- a/MetricsPlugin/CoreInterfaceExtensions.cs +++ b/MetricsPlugin/CoreInterfaceExtensions.cs @@ -16,6 +16,11 @@ namespace MetricsPlugin return Plugin(ci).CreateAccessForTarget(metricsContainer, scrapeTarget); } + public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IManyMetricScrapeTargets[] manyScrapeTargets) + { + return ci.GetMetricsFor(manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray()); + } + public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets) { var rc = ci.StartMetricsCollector(scrapeTargets); diff --git a/MetricsPlugin/MetricsAccess.cs b/MetricsPlugin/MetricsAccess.cs index beb5ed1..aa172f9 100644 --- a/MetricsPlugin/MetricsAccess.cs +++ b/MetricsPlugin/MetricsAccess.cs @@ -4,6 +4,7 @@ namespace MetricsPlugin { public interface IMetricsAccess { + string TargetName { get; } Metrics? GetAllMetrics(); MetricsSet GetMetric(string metricName); MetricsSet GetMetric(string metricName, TimeSpan timeout); @@ -18,19 +19,10 @@ namespace MetricsPlugin { this.query = query; this.target = target; + TargetName = target.Name; } - //public void AssertThat(string metricName, IResolveConstraint constraint, string message = "") - //{ - // AssertHelpers.RetryAssert(constraint, () => - // { - // var metricSet = GetMetricWithTimeout(metricName); - // var metricValue = metricSet.Values[0].Value; - - // log.Log($"{node.Name} metric '{metricName}' = {metricValue}"); - // return metricValue; - // }, message); - //} + public string TargetName { get; } public Metrics? GetAllMetrics() { diff --git a/MetricsPlugin/MetricsScrapeTarget.cs b/MetricsPlugin/MetricsScrapeTarget.cs index 6dff1be..e2e1979 100644 --- a/MetricsPlugin/MetricsScrapeTarget.cs +++ b/MetricsPlugin/MetricsScrapeTarget.cs @@ -4,20 +4,27 @@ namespace MetricsPlugin { public interface IMetricsScrapeTarget { + string Name { get; } string Ip { get; } int Port { get; } } + public interface IManyMetricScrapeTargets + { + IMetricsScrapeTarget[] ScrapeTargets { get; } + } + public class MetricsScrapeTarget : IMetricsScrapeTarget { - public MetricsScrapeTarget(string ip, int port) + public MetricsScrapeTarget(string ip, int port, string name) { Ip = ip; Port = port; + Name = name; } public MetricsScrapeTarget(RunningContainer container, int port) - : this(container.Pod.PodInfo.Ip, port) + : this(container.Pod.PodInfo.Ip, port, container.Name) { } @@ -26,6 +33,7 @@ namespace MetricsPlugin { } + public string Name { get; } public string Ip { get; } public int Port { get; } } diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 0b5156d..f9e76de 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -24,23 +24,23 @@ namespace Tests.BasicTests [Test] public void TwoMetricsExample() { - var rc = Ci.StartMetricsCollector(); + var group = Ci.SetupCodexNodes(2, s => s.EnableMetrics()); + var group2 = Ci.SetupCodexNodes(2, s => s.EnableMetrics()); - //var group = Ci.SetupCodexNodes(2, s => s.EnableMetrics()); - //var group2 = Ci.SetupCodexNodes(2, s => s.EnableMetrics()); + var primary = group[0]; + var secondary = group[1]; + var primary2 = group2[0]; + var secondary2 = group2[1]; - //var primary = group[0]; - //var secondary = group[1]; - //var primary2 = group2[0]; - //var secondary2 = group2[1]; + var metrics = Ci.GetMetricsFor(primary.MetricsScrapeTarget, primary2.MetricsScrapeTarget); - //primary.ConnectToPeer(secondary); - //primary2.ConnectToPeer(secondary2); + primary.ConnectToPeer(secondary); + primary2.ConnectToPeer(secondary2); - //Thread.Sleep(TimeSpan.FromMinutes(2)); + Thread.Sleep(TimeSpan.FromMinutes(2)); - //primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); - //primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + metrics[0].AssertThat("libp2p_peers", Is.EqualTo(1)); + metrics[1].AssertThat("libp2p_peers", Is.EqualTo(1)); } [Test] diff --git a/Tests/MetricsAccessExtensions.cs b/Tests/MetricsAccessExtensions.cs new file mode 100644 index 0000000..aa5da83 --- /dev/null +++ b/Tests/MetricsAccessExtensions.cs @@ -0,0 +1,22 @@ +using DistTestCore.Helpers; +using Logging; +using MetricsPlugin; +using NUnit.Framework.Constraints; + +namespace Tests +{ + public static class MetricsAccessExtensions + { + public static void AssertThat(this IMetricsAccess access, string metricName, IResolveConstraint constraint, ILog? log = null, string message = "") + { + AssertHelpers.RetryAssert(constraint, () => + { + var metricSet = access.GetMetric(metricName); + var metricValue = metricSet.Values[0].Value; + + if (log != null) log.Log($"{access.TargetName} metric '{metricName}' = {metricValue}"); + return metricValue; + }, message); + } + } +}