Merge branch 'feature/codex-net-deployer'

# Conflicts:
#	DistTestCore/Marketplace/MarketplaceAccess.cs

CodexNetDeployer/CodexNodeStarter.cs

@@ -20,7 +20,7 @@ namespace CodexNetDeployer
             this.workflowCreator = workflowCreator;
             this.lifecycle = lifecycle;
             this.gethResult = gethResult;
-            this.validatorsLeft = numberOfValidators;
+            validatorsLeft = numberOfValidators;
         }
 
         public RunningContainer? Start(int i)
@@ -86,6 +86,8 @@ namespace CodexNetDeployer
             var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
             marketplaceConfig.AccountIndexOverride = i;
             codexStart.MarketplaceConfig = marketplaceConfig;
+            codexStart.MetricsEnabled = config.RecordMetrics;
+
             if (config.BlockTTL != Configuration.SecondsIn1Day)
             {
                 codexStart.BlockTTL = config.BlockTTL;

CodexNetDeployer/Configuration.cs

@@ -1,7 +1,6 @@
 using ArgsUniform;
 using DistTestCore;
 using DistTestCore.Codex;
-using DistTestCore.Marketplace;
 
 namespace CodexNetDeployer
 {
@@ -9,15 +8,6 @@ namespace CodexNetDeployer
     {
         public const int SecondsIn1Day = 24 * 60 * 60;
 
-        [Uniform("codex-image", "ci", "CODEXIMAGE", true, "Docker image of Codex.")]
-        public string CodexImage { get; set; } = CodexContainerRecipe.DockerImage;
-
-        [Uniform("geth-image", "gi", "GETHIMAGE", true, "Docker image of Geth.")]
-        public string GethImage { get; set; } = GethContainerRecipe.DockerImage;
-
-        [Uniform("contracts-image", "oi", "CONTRACTSIMAGE", true, "Docker image of Codex Contracts.")]
-        public string ContractsImage { get; set; } = CodexContractsContainerRecipe.DockerImage;
-
         [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
         public string KubeConfigFile { get; set; } = "null";
 
@@ -54,6 +44,9 @@ namespace CodexNetDeployer
         [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
         public int BlockTTL { get; set; } = SecondsIn1Day;
 
+        [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
+        public bool RecordMetrics { get; set; } = false;
+       
         public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
 
         public List<string> Validate()

CodexNetDeployer/Deployer.cs

@@ -27,6 +27,7 @@ namespace CodexNetDeployer
             // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
             var setup = new CodexSetup(999, config.CodexLogLevel);
             setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
+            setup.MetricsEnabled = config.RecordMetrics;
 
             Log("Creating Geth instance and deploying contracts...");
             var gethStarter = new GethStarter(lifecycle, workflowCreator);
@@ -51,7 +52,9 @@ namespace CodexNetDeployer
                 if (container != null) codexContainers.Add(container);
             }
 
-            return new CodexDeployment(gethResults, codexContainers.ToArray(), CreateMetadata());
+            var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers);
+
+            return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
         }
 
         private (WorkflowCreator, TestLifecycle) CreateFacilities()
@@ -80,6 +83,15 @@ namespace CodexNetDeployer
             return (workflowCreator, lifecycle);
         }
 
+        private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
+        {
+            if (!setup.MetricsEnabled) return null;
+
+            Log("Starting metrics service...");
+            var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray());
+            return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
+        }
+
         private string? GetKubeConfig(string kubeConfigFile)
         {
             if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
@@ -89,9 +101,6 @@ namespace CodexNetDeployer
         private DeploymentMetadata CreateMetadata()
         {
             return new DeploymentMetadata(
-                codexImage: config.CodexImage,
-                gethImage: config.GethImage,
-                contractsImage: config.ContractsImage,
                 kubeNamespace: config.KubeNamespace,
                 numberOfCodexNodes: config.NumberOfCodexNodes!.Value,
                 numberOfValidators: config.NumberOfValidators!.Value,

CodexNetDeployer/Program.cs

@@ -29,6 +29,13 @@ public class Program
             return;
         }
 
+        if (!args.Any(a => a == "-y"))
+        {
+            Console.WriteLine("Does the above config look good? [y/n]");
+            if (Console.ReadLine()!.ToLowerInvariant() != "y") return;
+            Console.WriteLine("I think so too.");
+        }
+
         var deployer = new Deployer(config);
         var deployment = deployer.Deploy();
 

ContinuousTests/StartupChecker.cs

@@ -64,9 +64,9 @@ namespace ContinuousTests
             {
                 cancelToken.ThrowIfCancellationRequested();
 
-                log.Log($"Checking '{n.Address.Host}'...");
+                log.Log($"Checking {n.Container.Name} @ '{n.Address.Host}:{n.Address.Port}'...");
 
-                if (EnsureOnline(n))
+                if (EnsureOnline(log, n))
                 {
                     log.Log("OK");
                 }
@@ -82,12 +82,14 @@ namespace ContinuousTests
             }
         }
 
-        private bool EnsureOnline(CodexAccess n)
+        private bool EnsureOnline(BaseLog log, CodexAccess n)
         {
             try
             {
                 var info = n.GetDebugInfo();
                 if (info == null || string.IsNullOrEmpty(info.id)) return false;
+
+                log.Log($"Codex version: '{info.codex.version}' revision: '{info.codex.revision}'");
             }
             catch
             {

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -33,7 +33,7 @@ namespace DistTestCore.Codex
             AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
 
             // This makes the node announce itself to its local (pod) IP address.
-            AddEnvVar("CODEX_NAT_ADDR", "$(hostname --ip-address)");
+            AddEnvVar("NAT_IP_AUTO", "true");
 
             var listenPort = AddInternalPort();
             AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");

DistTestCore/Codex/CodexDeployment.cs

@@ -5,26 +5,25 @@ namespace DistTestCore.Codex
 {
     public class CodexDeployment
     {
-        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, DeploymentMetadata metadata)
+        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata)
         {
             GethStartResult = gethStartResult;
             CodexContainers = codexContainers;
+            PrometheusContainer = prometheusContainer;
             Metadata = metadata;
         }
 
         public GethStartResult GethStartResult { get; }
         public RunningContainer[] CodexContainers { get; }
+        public RunningContainer? PrometheusContainer { get; }
         public DeploymentMetadata Metadata { get; }
     }
 
     public class DeploymentMetadata
     {
-        public DeploymentMetadata(string codexImage, string gethImage, string contractsImage, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration)
+        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration)
         {
             DeployDateTimeUtc = DateTime.UtcNow;
-            CodexImage = codexImage;
-            GethImage = gethImage;
-            ContractsImage = contractsImage;
             KubeNamespace = kubeNamespace;
             NumberOfCodexNodes = numberOfCodexNodes;
             NumberOfValidators = numberOfValidators;
@@ -36,10 +35,7 @@ namespace DistTestCore.Codex
             MaxDuration = maxDuration;
         }
 
-        public string CodexImage { get; }
         public DateTime DeployDateTimeUtc { get; }
-        public string GethImage { get; }
-        public string ContractsImage { get; }
         public string KubeNamespace { get; }
         public int NumberOfCodexNodes { get; }
         public int NumberOfValidators { get; }

DistTestCore/CodexStarter.cs

@@ -1,5 +1,6 @@
 using DistTestCore.Codex;
 using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
 using KubernetesWorkflow;
 using Logging;
 
@@ -23,7 +24,7 @@ namespace DistTestCore
             var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
             var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
 
-            var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers);
+            var metricAccessFactory = CollectMetrics(codexSetup, containers);
             
             var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
 
@@ -57,6 +58,14 @@ namespace DistTestCore
             workflow.DownloadContainerLog(container, logHandler);
         }
 
+        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers containers)
+        {
+            if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
+
+            var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
+            return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
+        }
+
         private StartupConfig CreateStartupConfig(GethStartResult gethStartResult, CodexSetup codexSetup)
         {
             var startupConfig = new StartupConfig();

DistTestCore/PrometheusStarter.cs

@@ -12,10 +12,8 @@ namespace DistTestCore
         {
         }
 
-        public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers)
+        public RunningContainers CollectMetricsFor(RunningContainers containers)
         {
-            if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
-
             LogStart($"Starting metrics server for {containers.Describe()}");
             var startupConfig = new StartupConfig();
             startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers)));
@@ -26,7 +24,7 @@ namespace DistTestCore
 
             LogEnd("Metrics server started.");
 
-            return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
+            return runningContainers;
         }
 
         private string GeneratePrometheusConfig(RunningContainer[] nodes)

Tests/BasicTests/ThreeClientTest.cs

@@ -0,0 +1,24 @@
+using DistTestCore;
+using NUnit.Framework;
+
+namespace Tests.BasicTests
+{
+    [TestFixture]
+    public class ThreeClientTest : AutoBootstrapDistTest
+    {
+        [Test]
+        public void ThreeClient()
+        {
+            var primary = SetupCodexNode();
+            var secondary = SetupCodexNode();
+
+            var testFile = GenerateTestFile(10.MB());
+
+            var contentId = primary.UploadFile(testFile);
+
+            var downloadedFile = secondary.DownloadContent(contentId);
+
+            testFile.AssertIsEqual(downloadedFile);
+        }
+    }
+}