diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 999ba14..9478a98 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -20,7 +20,7 @@ namespace CodexNetDeployer this.workflowCreator = workflowCreator; this.lifecycle = lifecycle; this.gethResult = gethResult; - this.validatorsLeft = numberOfValidators; + validatorsLeft = numberOfValidators; } public RunningContainer? Start(int i) @@ -86,6 +86,8 @@ namespace CodexNetDeployer var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0); marketplaceConfig.AccountIndexOverride = i; codexStart.MarketplaceConfig = marketplaceConfig; + codexStart.MetricsEnabled = config.RecordMetrics; + if (config.BlockTTL != Configuration.SecondsIn1Day) { codexStart.BlockTTL = config.BlockTTL; diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 648e5bb..b030618 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -1,7 +1,6 @@ using ArgsUniform; using DistTestCore; using DistTestCore.Codex; -using DistTestCore.Marketplace; namespace CodexNetDeployer { @@ -9,15 +8,6 @@ namespace CodexNetDeployer { public const int SecondsIn1Day = 24 * 60 * 60; - [Uniform("codex-image", "ci", "CODEXIMAGE", true, "Docker image of Codex.")] - public string CodexImage { get; set; } = CodexContainerRecipe.DockerImage; - - [Uniform("geth-image", "gi", "GETHIMAGE", true, "Docker image of Geth.")] - public string GethImage { get; set; } = GethContainerRecipe.DockerImage; - - [Uniform("contracts-image", "oi", "CONTRACTSIMAGE", true, "Docker image of Codex Contracts.")] - public string ContractsImage { get; set; } = CodexContractsContainerRecipe.DockerImage; - [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] public string KubeConfigFile { get; set; } = "null"; @@ -54,6 +44,9 @@ namespace CodexNetDeployer [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")] public int BlockTTL { get; set; } = SecondsIn1Day; + [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")] + public bool RecordMetrics { get; set; } = false; + public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster; public List Validate() diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 65cbb06..e7180a6 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -27,6 +27,7 @@ namespace CodexNetDeployer // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes. var setup = new CodexSetup(999, config.CodexLogLevel); setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens()); + setup.MetricsEnabled = config.RecordMetrics; Log("Creating Geth instance and deploying contracts..."); var gethStarter = new GethStarter(lifecycle, workflowCreator); @@ -51,7 +52,9 @@ namespace CodexNetDeployer if (container != null) codexContainers.Add(container); } - return new CodexDeployment(gethResults, codexContainers.ToArray(), CreateMetadata()); + var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers); + + return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata()); } private (WorkflowCreator, TestLifecycle) CreateFacilities() @@ -80,6 +83,15 @@ namespace CodexNetDeployer return (workflowCreator, lifecycle); } + private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) + { + if (!setup.MetricsEnabled) return null; + + Log("Starting metrics service..."); + var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray()); + return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single(); + } + private string? GetKubeConfig(string kubeConfigFile) { if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null; @@ -89,9 +101,6 @@ namespace CodexNetDeployer private DeploymentMetadata CreateMetadata() { return new DeploymentMetadata( - codexImage: config.CodexImage, - gethImage: config.GethImage, - contractsImage: config.ContractsImage, kubeNamespace: config.KubeNamespace, numberOfCodexNodes: config.NumberOfCodexNodes!.Value, numberOfValidators: config.NumberOfValidators!.Value, diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index b50c159..234ce78 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -29,6 +29,13 @@ public class Program return; } + if (!args.Any(a => a == "-y")) + { + Console.WriteLine("Does the above config look good? [y/n]"); + if (Console.ReadLine()!.ToLowerInvariant() != "y") return; + Console.WriteLine("I think so too."); + } + var deployer = new Deployer(config); var deployment = deployer.Deploy(); diff --git a/ContinuousTests/StartupChecker.cs b/ContinuousTests/StartupChecker.cs index 8e9357a..a2e1e1f 100644 --- a/ContinuousTests/StartupChecker.cs +++ b/ContinuousTests/StartupChecker.cs @@ -64,9 +64,9 @@ namespace ContinuousTests { cancelToken.ThrowIfCancellationRequested(); - log.Log($"Checking '{n.Address.Host}'..."); + log.Log($"Checking {n.Container.Name} @ '{n.Address.Host}:{n.Address.Port}'..."); - if (EnsureOnline(n)) + if (EnsureOnline(log, n)) { log.Log("OK"); } @@ -82,12 +82,14 @@ namespace ContinuousTests } } - private bool EnsureOnline(CodexAccess n) + private bool EnsureOnline(BaseLog log, CodexAccess n) { try { var info = n.GetDebugInfo(); if (info == null || string.IsNullOrEmpty(info.id)) return false; + + log.Log($"Codex version: '{info.codex.version}' revision: '{info.codex.revision}'"); } catch { diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 64f2177..480503b 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -33,7 +33,7 @@ namespace DistTestCore.Codex AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); // This makes the node announce itself to its local (pod) IP address. - AddEnvVar("CODEX_NAT_ADDR", "$(hostname --ip-address)"); + AddEnvVar("NAT_IP_AUTO", "true"); var listenPort = AddInternalPort(); AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); diff --git a/DistTestCore/Codex/CodexDeployment.cs b/DistTestCore/Codex/CodexDeployment.cs index 37db831..52192dc 100644 --- a/DistTestCore/Codex/CodexDeployment.cs +++ b/DistTestCore/Codex/CodexDeployment.cs @@ -5,26 +5,25 @@ namespace DistTestCore.Codex { public class CodexDeployment { - public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, DeploymentMetadata metadata) + public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata) { GethStartResult = gethStartResult; CodexContainers = codexContainers; + PrometheusContainer = prometheusContainer; Metadata = metadata; } public GethStartResult GethStartResult { get; } public RunningContainer[] CodexContainers { get; } + public RunningContainer? PrometheusContainer { get; } public DeploymentMetadata Metadata { get; } } public class DeploymentMetadata { - public DeploymentMetadata(string codexImage, string gethImage, string contractsImage, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration) + public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration) { DeployDateTimeUtc = DateTime.UtcNow; - CodexImage = codexImage; - GethImage = gethImage; - ContractsImage = contractsImage; KubeNamespace = kubeNamespace; NumberOfCodexNodes = numberOfCodexNodes; NumberOfValidators = numberOfValidators; @@ -36,10 +35,7 @@ namespace DistTestCore.Codex MaxDuration = maxDuration; } - public string CodexImage { get; } public DateTime DeployDateTimeUtc { get; } - public string GethImage { get; } - public string ContractsImage { get; } public string KubeNamespace { get; } public int NumberOfCodexNodes { get; } public int NumberOfValidators { get; } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 627bfb9..144d55f 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using DistTestCore.Marketplace; +using DistTestCore.Metrics; using KubernetesWorkflow; using Logging; @@ -23,7 +24,7 @@ namespace DistTestCore var startupConfig = CreateStartupConfig(gethStartResult, codexSetup); var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location); - var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers); + var metricAccessFactory = CollectMetrics(codexSetup, containers); var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory); @@ -57,6 +58,14 @@ namespace DistTestCore workflow.DownloadContainerLog(container, logHandler); } + private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers containers) + { + if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); + + var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers); + return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers); + } + private StartupConfig CreateStartupConfig(GethStartResult gethStartResult, CodexSetup codexSetup) { var startupConfig = new StartupConfig(); diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index 64edae9..a0fc9a4 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -12,10 +12,8 @@ namespace DistTestCore { } - public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers) + public RunningContainers CollectMetricsFor(RunningContainers containers) { - if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); - LogStart($"Starting metrics server for {containers.Describe()}"); var startupConfig = new StartupConfig(); startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers))); @@ -26,7 +24,7 @@ namespace DistTestCore LogEnd("Metrics server started."); - return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers); + return runningContainers; } private string GeneratePrometheusConfig(RunningContainer[] nodes) diff --git a/Tests/BasicTests/ThreeClientTest.cs b/Tests/BasicTests/ThreeClientTest.cs new file mode 100644 index 0000000..78ef25e --- /dev/null +++ b/Tests/BasicTests/ThreeClientTest.cs @@ -0,0 +1,24 @@ +using DistTestCore; +using NUnit.Framework; + +namespace Tests.BasicTests +{ + [TestFixture] + public class ThreeClientTest : AutoBootstrapDistTest + { + [Test] + public void ThreeClient() + { + var primary = SetupCodexNode(); + var secondary = SetupCodexNode(); + + var testFile = GenerateTestFile(10.MB()); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = secondary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + } +}