diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 65db0e75..9478a989 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -86,6 +86,8 @@ namespace CodexNetDeployer var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); marketplaceConfig.AccountIndexOverride = i; codexStart.MarketplaceConfig = marketplaceConfig; + codexStart.MetricsEnabled = config.RecordMetrics; + if (config.BlockTTL != Configuration.SecondsIn1Day) { codexStart.BlockTTL = config.BlockTTL; diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 89fce7e0..f3cc7937 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -2,6 +2,7 @@ using DistTestCore; using DistTestCore.Codex; using DistTestCore.Marketplace; +using DistTestCore.Metrics; namespace CodexNetDeployer { @@ -18,6 +19,9 @@ namespace CodexNetDeployer [Uniform("contracts-image", "oi", "CONTRACTSIMAGE", true, "Docker image of Codex Contracts.")] public string ContractsImage { get; set; } = CodexContractsContainerRecipe.DockerImage; + [Uniform("metrics-image", "mi", "METRICSIMAGE", true, "Docker image of Prometheus.")] + public string MetricsImage { get; set; } = PrometheusContainerRecipe.DockerImage; + [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] public string KubeConfigFile { get; set; } = "null"; diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 04461e9d..04c39d1b 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -52,12 +52,9 @@ namespace CodexNetDeployer if (container != null) codexContainers.Add(container); } - if (setup.MetricsEnabled) - { - StartMetricsService(lifecycle, setup, codexContainers); - } + var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers); - return new CodexDeployment(gethResults, codexContainers.ToArray(), CreateMetadata()); + return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata()); } private (WorkflowCreator, TestLifecycle) CreateFacilities() @@ -86,11 +83,13 @@ namespace CodexNetDeployer return (workflowCreator, lifecycle); } - private void StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) + private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) { + if (!setup.MetricsEnabled) return null; + Log("Starting metrics service..."); var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray()); - lifecycle.PrometheusStarter.CollectMetricsFor(setup, runningContainers); + return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single(); } private string? GetKubeConfig(string kubeConfigFile) @@ -105,6 +104,7 @@ namespace CodexNetDeployer codexImage: config.CodexImage, gethImage: config.GethImage, contractsImage: config.ContractsImage, + prometheusImage: config.MetricsImage, kubeNamespace: config.KubeNamespace, numberOfCodexNodes: config.NumberOfCodexNodes!.Value, numberOfValidators: config.NumberOfValidators!.Value, diff --git a/DistTestCore/Codex/CodexDeployment.cs b/DistTestCore/Codex/CodexDeployment.cs index 37db8310..39aa6895 100644 --- a/DistTestCore/Codex/CodexDeployment.cs +++ b/DistTestCore/Codex/CodexDeployment.cs @@ -5,26 +5,29 @@ namespace DistTestCore.Codex { public class CodexDeployment { - public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, DeploymentMetadata metadata) + public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata) { GethStartResult = gethStartResult; CodexContainers = codexContainers; + PrometheusContainer = prometheusContainer; Metadata = metadata; } public GethStartResult GethStartResult { get; } public RunningContainer[] CodexContainers { get; } + public RunningContainer? PrometheusContainer { get; } public DeploymentMetadata Metadata { get; } } public class DeploymentMetadata { - public DeploymentMetadata(string codexImage, string gethImage, string contractsImage, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration) + public DeploymentMetadata(string codexImage, string gethImage, string contractsImage, string prometheusImage, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration) { DeployDateTimeUtc = DateTime.UtcNow; CodexImage = codexImage; GethImage = gethImage; ContractsImage = contractsImage; + PrometheusImage = prometheusImage; KubeNamespace = kubeNamespace; NumberOfCodexNodes = numberOfCodexNodes; NumberOfValidators = numberOfValidators; @@ -40,6 +43,7 @@ namespace DistTestCore.Codex public DateTime DeployDateTimeUtc { get; } public string GethImage { get; } public string ContractsImage { get; } + public string PrometheusImage { get; } public string KubeNamespace { get; } public int NumberOfCodexNodes { get; } public int NumberOfValidators { get; } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 627bfb96..144d55f4 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using DistTestCore.Marketplace; +using DistTestCore.Metrics; using KubernetesWorkflow; using Logging; @@ -23,7 +24,7 @@ namespace DistTestCore var startupConfig = CreateStartupConfig(gethStartResult, codexSetup); var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location); - var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers); + var metricAccessFactory = CollectMetrics(codexSetup, containers); var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory); @@ -57,6 +58,14 @@ namespace DistTestCore workflow.DownloadContainerLog(container, logHandler); } + private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers containers) + { + if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); + + var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers); + return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers); + } + private StartupConfig CreateStartupConfig(GethStartResult gethStartResult, CodexSetup codexSetup) { var startupConfig = new StartupConfig(); diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index 64edae91..a0fc9a41 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -12,10 +12,8 @@ namespace DistTestCore { } - public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers) + public RunningContainers CollectMetricsFor(RunningContainers containers) { - if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); - LogStart($"Starting metrics server for {containers.Describe()}"); var startupConfig = new StartupConfig(); startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers))); @@ -26,7 +24,7 @@ namespace DistTestCore LogEnd("Metrics server started."); - return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers); + return runningContainers; } private string GeneratePrometheusConfig(RunningContainer[] nodes)