Makes sure codex nodes are always started in their own pods.

CodexNetDeployer/Deployer.cs

@@ -88,7 +88,7 @@ namespace CodexNetDeployer
             if (!setup.MetricsEnabled) return null;
 
             Log("Starting metrics service...");
-            var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray());
+            var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
             return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
         }
 

ContinuousTests/reports/CodexTestNetReport-July2023.md

@@ -0,0 +1,38 @@
+# Codex Continuous Test-net Report
+Date: 02-08-2023
+
+Report for: 07-2023
+
+
+## Test-net Status
+- Start of month: Offline - faulted
+- End of month: Offline - faulted
+
+(Faulted: Tests fail with such frequency that the information gathered does not justify the cost of leaving the test-net running.)
+
+## Deployment Configuration
+Continous Test-net is deployed to the kubernetes cluster with the following configuration:
+
+5x Codex Nodes:
+- Log-level: Trace
+- Storage quota: 2048 MB
+- Storage sell: 1024 MB
+- Min price: 1024
+- Max collateral: 1024
+- Max duration: 3600000 seconds
+- Block-TTL: 120 seconds
+
+3 of these 5 nodes have:
+- Validator: true
+
+Kubernetes namespace: 'codex-continuous-tests'
+
+## Test Overview
+| Changes        | Test                | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | Status      | Results                                                                              |
+|----------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|--------------------------------------------------------------------------------------|
+| New in 07-2023 | Two-client test     | Every 30 seconds, two nodes are chosen at random. A 10 MB file is generated and uploaded to one. 10 seconds later, it is downloaded from the other. File contents are asserted to be equal.                                                                                                                                                                                                                                                                                                                                                                                    | Faulted     | Test reliably fails after 30 to 45 minutes. Both upload and download failures occur. |
+| New in 07-2023 | Transient-node test | Every 1 minute, a new, transient Codex node is started and bootstrapped against a random node of the test net. A 10 MB file is generated and uploaded to the transient node. The file is then downloaded from a second random test net node and file equality is asserted. After that, the transient node is shut down. 30 seconds later, a new transient Codex node is started and bootstrapped against a third (guaranteed different from first and second) node from the test net. The same file is then downloaded from the new transient node. File equality is asserted. | Not running | Test was not run because the previous more rudamentory test has faulted.             |
+
+## Action Points
+- Codex logs (from these long-running containers) are often incomplete when downloaded after a test failure. A reliable way of maintaining these logs is needed. The logs can quickly explode in size, even for test-nets that run only for a few hours.
+- The distributed testing setup can and has been used to reproduce the failure of the Two-client test in a local environment. Investigation is on-going.

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -5,7 +5,7 @@ namespace DistTestCore.Codex
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:latest";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-7efa917";
 
         public const string MetricsPortTag = "metrics_port";
         public const string DiscoveryPortTag = "discovery-port";

DistTestCore/CodexNodeGroup.cs

@@ -14,12 +14,12 @@ namespace DistTestCore
     {
         private readonly TestLifecycle lifecycle;
 
-        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
         {
             this.lifecycle = lifecycle;
             Setup = setup;
             Containers = containers;
-            Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
+            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
             Version = new CodexDebugVersionResponse();
         }
 
@@ -45,7 +45,7 @@ namespace DistTestCore
         }
 
         public CodexSetup Setup { get; private set; }
-        public RunningContainers Containers { get; private set; }
+        public RunningContainers[] Containers { get; private set; }
         public OnlineCodexNode[] Nodes { get; private set; }
         public CodexDebugVersionResponse Version { get; private set; }
 

DistTestCore/CodexStarter.cs

@@ -32,12 +32,12 @@ namespace DistTestCore
             var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
             lifecycle.SetCodexVersion(group.Version);
 
-            var podInfo = group.Containers.RunningPod.PodInfo;
+            var nl = Environment.NewLine;
+            var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
             LogEnd($"Started {codexSetup.NumberOfNodes} nodes " +
-                $"of image '{containers.Containers.First().Recipe.Image}' " +
-                $"and version '{group.Version}' " +
-                $"at location '{podInfo.K8SNodeName}'={podInfo.Ip}. " +
-                $"They are: {group.Describe()}");
+                $"of image '{containers.Containers().First().Recipe.Image}' " +
+                $"and version '{group.Version}'{nl}" +
+                podInfos);
             LogSeparator();
 
             return group;
@@ -47,7 +47,7 @@ namespace DistTestCore
         {
             LogStart($"Stopping {group.Describe()}...");
             var workflow = CreateWorkflow();
-            workflow.Stop(group.Containers);
+            foreach (var c in group.Containers) workflow.Stop(c);
             RunningGroups.Remove(group);
             LogEnd("Stopped.");
         }
@@ -66,7 +66,7 @@ namespace DistTestCore
             workflow.DownloadContainerLog(container, logHandler);
         }
 
-        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers containers)
+        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
         {
             if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
 
@@ -83,20 +83,42 @@ namespace DistTestCore
             return startupConfig;
         }
 
-        private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
+        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
         {
-            var workflow = CreateWorkflow();
-            return workflow.Start(numberOfNodes, location, new CodexContainerRecipe(), startupConfig);
+            var result = new List<RunningContainers>();
+            var recipe = new CodexContainerRecipe();
+            for (var i = 0; i < numberOfNodes; i++)
+            {
+                var workflow = CreateWorkflow();
+                result.Add(workflow.Start(1, location, recipe, startupConfig));
+            }
+            return result.ToArray();
         }
 
-        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory)
+        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
             RunningGroups.Add(group);
-            Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
+
+            try
+            {
+                Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
+            }
+            catch
+            {
+                CodexNodesNotOnline(runningContainers);
+                throw;
+            }
+
             return group;
         }
 
+        private void CodexNodesNotOnline(RunningContainers[] runningContainers)
+        {
+            Log("Codex nodes failed to start");
+            foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
+        }
+
         private StartupWorkflow CreateWorkflow()
         {
             return workflowCreator.CreateWorkflow();

DistTestCore/OnlineCodexNode.cs

@@ -135,14 +135,9 @@ namespace DistTestCore
             var multiAddress = peerInfo.addrs.First();
             // Todo: Is there a case where First address in list is not the way?
 
-            if (Group == peer.Group)
-            {
-                return multiAddress;
-            }
-
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip);
+            return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip);
         }
 
         private void DownloadToFile(string contentId, TestFile file)

DistTestCore/PrometheusStarter.cs

@@ -12,11 +12,11 @@ namespace DistTestCore
         {
         }
 
-        public RunningContainers CollectMetricsFor(RunningContainers containers)
+        public RunningContainers CollectMetricsFor(RunningContainers[] containers)
         {
             LogStart($"Starting metrics server for {containers.Describe()}");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers)));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers())));
 
             var workflow = workflowCreator.CreateWorkflow();
             var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);

KubernetesWorkflow/RunningContainers.cs

@@ -40,4 +40,17 @@ namespace KubernetesWorkflow
         public Address ClusterExternalAddress { get; }
         public Address ClusterInternalAddress { get; }
     }
+
+    public static class RunningContainersExtensions
+    {
+        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
+        {
+            return runningContainers.SelectMany(c => c.Containers).ToArray();
+        }
+
+        public static string Describe(this RunningContainers[] runningContainers)
+        {
+            return string.Join(",", runningContainers.Select(c => c.Describe()));
+        }
+    }
 }

Tests/BasicTests/ExampleTests.cs

@@ -1,9 +1,5 @@
 using DistTestCore;
-using DistTestCore.Codex;
-using DistTestCore.Marketplace;
-using Newtonsoft.Json;
 using NUnit.Framework;
-using Utils;
 
 namespace Tests.BasicTests
 {