this works

CodexDistTestCore/ActiveDeployment.cs

@@ -2,18 +2,18 @@
 
 namespace CodexDistTestCore
 {
-    public class ActiveNode
+    public class ActiveDeployment
     {
-        public ActiveNode(OfflineCodexNode origin, int port, int orderNumber)
+        public ActiveDeployment(OfflineCodexNodes origin, int orderNumber, CodexNodeContainer[] containers)
         {
             Origin = origin;
+            Containers = containers;
             SelectorName = orderNumber.ToString().PadLeft(6, '0');
-            Port = port;
         }
 
-        public OfflineCodexNode Origin { get; }
+        public OfflineCodexNodes Origin { get; }
+        public CodexNodeContainer[] Containers { get; }
         public string SelectorName { get; }
-        public int Port { get; }
         public V1Deployment? Deployment { get; set; }
         public V1Service? Service { get; set; }
         public List<string> ActivePodNames { get; } = new List<string>();
@@ -41,21 +41,9 @@ namespace CodexDistTestCore
             return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + SelectorName } };
         }
 
-        public string GetContainerPortName()
-        {
-            //Caution, was: "codex-api-port" + SelectorName
-            //but string length causes 'UnprocessableEntity' exception in k8s.
-            return "api-" + SelectorName;
-        }
-
-        public string GetContainerName()
-        {
-            return "codex-test-node";
-        }
-
         public string Describe()
         {
-            return $"CodexNode{SelectorName}-Port:{Port}-{Origin.Describe()}";
+            return $"CodexNode{SelectorName}-{Origin.Describe()}";
         }
     }
 }

CodexDistTestCore/CodexDockerImage.cs

@@ -14,10 +14,10 @@ namespace CodexDistTestCore
             return "b20483";
         }
 
-        public List<V1EnvVar> CreateEnvironmentVariables(OfflineCodexNode node)
+        public List<V1EnvVar> CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer environment)
         {
             var formatter = new EnvFormatter();
-            formatter.Create(node);
+            formatter.Create(node, environment);
             return formatter.Result;
         }
 
@@ -25,8 +25,13 @@ namespace CodexDistTestCore
         {
             public List<V1EnvVar> Result { get; } = new List<V1EnvVar>();
 
-            public void Create(OfflineCodexNode node)
+            public void Create(OfflineCodexNodes node, CodexNodeContainer environment)
             {
+                AddVar("API_PORT", environment.ApiPort.ToString());
+                AddVar("DATA_DIR", environment.DataDir);
+                AddVar("DISC_PORT", environment.DiscoveryPort.ToString());
+                AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{environment.ListenPort}");
+
                 if (node.BootstrapNode != null)
                 {
                     var debugInfo = node.BootstrapNode.GetDebugInfo();

CodexDistTestCore/CodexNodeContainer.cs

@@ -0,0 +1,45 @@
+namespace CodexDistTestCore
+{
+    public class CodexNodeContainer
+    {
+        public CodexNodeContainer(string name, int servicePort, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir)
+        {
+            Name = name;
+            ServicePort = servicePort;
+            ApiPort = apiPort;
+            ContainerPortName = containerPortName;
+            DiscoveryPort = discoveryPort;
+            ListenPort = listenPort;
+            DataDir = dataDir;
+        }
+
+        public string Name { get; }
+        public int ServicePort { get; }
+        public int ApiPort { get; }
+        public string ContainerPortName { get; }
+        public int DiscoveryPort { get; }
+        public int ListenPort { get; }
+        public string DataDir { get; }
+    }
+
+    public class CodexNodeContainerFactory
+    {
+        private readonly NumberSource containerNameSource = new NumberSource(1);
+        private readonly NumberSource servicePortSource = new NumberSource(30001);
+        private readonly NumberSource codexPortSource = new NumberSource(8080);
+
+        public CodexNodeContainer CreateNext()
+        {
+            var n = containerNameSource.GetNextNumber();
+            return new CodexNodeContainer(
+                name: $"codex-node{n}",
+                servicePort: servicePortSource.GetNextNumber(),
+                apiPort: codexPortSource.GetNextNumber(),
+                containerPortName: $"api-{n}",
+                discoveryPort: codexPortSource.GetNextNumber(),
+                listenPort: codexPortSource.GetNextNumber(),
+                dataDir: $"datadir{n}"
+            );
+        }
+    }
+}

CodexDistTestCore/DistTest.cs

@@ -56,9 +56,9 @@ namespace CodexDistTestCore
             return fileManager.GenerateTestFile(size);
         }
 
-        public IOfflineCodexNode SetupCodexNode()
+        public IOfflineCodexNodes SetupCodexNodes(int numberOfNodes)
         {
-            return new OfflineCodexNode(k8sManager);
+            return new OfflineCodexNodes(k8sManager, numberOfNodes);
         }
     }
 

CodexDistTestCore/K8sManager.cs

@@ -6,16 +6,16 @@ namespace CodexDistTestCore
 {
     public interface IK8sManager
     {
-        IOnlineCodexNode BringOnline(OfflineCodexNode node);
+        IOnlineCodexNodes BringOnline(OfflineCodexNodes node);
-        IOfflineCodexNode BringOffline(IOnlineCodexNode node);
+        IOfflineCodexNodes BringOffline(IOnlineCodexNodes node);
     }
 
     public class K8sManager : IK8sManager
     {
         public const string K8sNamespace = "codex-test-namespace";
         private readonly CodexDockerImage dockerImage = new CodexDockerImage();
-        private readonly NumberSource numberSource = new NumberSource();
+        private readonly NumberSource activeDeploymentOrderNumberSource = new NumberSource(0);
-        private readonly Dictionary<OnlineCodexNode, ActiveNode> activeNodes = new Dictionary<OnlineCodexNode, ActiveNode>();
+        private readonly Dictionary<OnlineCodexNodes, ActiveDeployment> activeDeployments = new Dictionary<OnlineCodexNodes, ActiveDeployment>();
         private readonly List<string> knownActivePodNames = new List<string>();
         private readonly IFileManager fileManager;
 
@@ -24,26 +24,38 @@ namespace CodexDistTestCore
             this.fileManager = fileManager;
         }
 
-        public IOnlineCodexNode BringOnline(OfflineCodexNode node)
+        public IOnlineCodexNodes BringOnline(OfflineCodexNodes node)
         {
             var client = CreateClient();
-
             EnsureTestNamespace(client);
 
-            var activeNode = new ActiveNode(node, numberSource.GetFreePort(), numberSource.GetNodeOrderNumber());
+            var factory = new CodexNodeContainerFactory();
-            var codexNode = new OnlineCodexNode(this, fileManager, activeNode.Port);
+            var list = new List<OnlineCodexNode>();
-            activeNodes.Add(codexNode, activeNode);
+            var containers = new List<CodexNodeContainer>();
+            for (var i = 0; i < node.NumberOfNodes; i++)
+            {
+                var container = factory.CreateNext();
+                containers.Add(container);
 
-            CreateDeployment(activeNode, client, node);
+                var codexNode = new OnlineCodexNode(fileManager, container);
-            CreateService(activeNode, client);
+                list.Add(codexNode);
+            }
 
-            WaitUntilOnline(activeNode, client);
+            var activeDeployment = new ActiveDeployment(node, activeDeploymentOrderNumberSource.GetNextNumber(), containers.ToArray());
-            TestLog.Log($"{activeNode.Describe()} online.");
 
-            return codexNode;
+            var result = new OnlineCodexNodes(this, list.ToArray());
+            activeDeployments.Add(result, activeDeployment);
+
+            CreateDeployment(activeDeployment, client, node);
+            CreateService(activeDeployment, client);
+
+            WaitUntilOnline(activeDeployment, client);
+            TestLog.Log($"{node.NumberOfNodes} Codex nodes online.");
+
+            return result;
         }
 
-        public IOfflineCodexNode BringOffline(IOnlineCodexNode node)
+        public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node)
         {
             var client = CreateClient();
 
@@ -70,7 +82,7 @@ namespace CodexDistTestCore
         public void FetchAllPodsLogs(Action<string, string, Stream> onLog)
         {
             var client = CreateClient();
-            foreach (var node in activeNodes.Values)
+            foreach (var node in activeDeployments.Values)
             {
                 var nodeDescription = node.Describe();
                 foreach (var podName in node.ActivePodNames)
@@ -81,7 +93,7 @@ namespace CodexDistTestCore
             }
         }
 
-        private void BringOffline(ActiveNode activeNode, Kubernetes client)
+        private void BringOffline(ActiveDeployment activeNode, Kubernetes client)
         {
             DeleteDeployment(activeNode, client);
             DeleteService(activeNode, client);
@@ -89,7 +101,7 @@ namespace CodexDistTestCore
 
         #region Waiting
 
-        private void WaitUntilOnline(ActiveNode activeNode, Kubernetes client)
+        private void WaitUntilOnline(ActiveDeployment activeNode, Kubernetes client)
         {
             WaitUntil(() =>
             {
@@ -100,7 +112,7 @@ namespace CodexDistTestCore
             AssignActivePodNames(activeNode, client);
         }
 
-        private void AssignActivePodNames(ActiveNode activeNode, Kubernetes client)
+        private void AssignActivePodNames(ActiveDeployment activeNode, Kubernetes client)
         {
             var pods = client.ListNamespacedPod(K8sNamespace);
             var podNames = pods.Items.Select(p => p.Name());
@@ -154,33 +166,40 @@ namespace CodexDistTestCore
 
         #region Service management
 
-        private void CreateService(ActiveNode node, Kubernetes client)
+        private void CreateService(ActiveDeployment activeDeployment, Kubernetes client)
         {
             var serviceSpec = new V1Service
             {
                 ApiVersion = "v1",
-                Metadata = node.GetServiceMetadata(),
+                Metadata = activeDeployment.GetServiceMetadata(),
                 Spec = new V1ServiceSpec
                 {
                     Type = "NodePort",
-                    Selector = node.GetSelector(),
+                    Selector = activeDeployment.GetSelector(),
-                    Ports = new List<V1ServicePort>
+                    Ports = CreateServicePorts(activeDeployment)
-                    {
-                        new V1ServicePort
-                        {
-                            Protocol = "TCP",
-                            Port = 8080,
-                            TargetPort = node.GetContainerPortName(),
-                            NodePort = node.Port
-                        }
-                    }
                 }
             };
 
-            node.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
+            activeDeployment.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
         }
 
-        private void DeleteService(ActiveNode node, Kubernetes client)
+        private List<V1ServicePort> CreateServicePorts(ActiveDeployment activeDeployment)
+        {
+            var result = new List<V1ServicePort>();
+            foreach (var container in activeDeployment.Containers)
+            {
+                result.Add(new V1ServicePort
+                {
+                    Protocol = "TCP",
+                    Port = 8080,
+                    TargetPort = container.ContainerPortName,
+                    NodePort = container.ServicePort
+                });
+            }
+            return result;
+        }
+
+        private void DeleteService(ActiveDeployment node, Kubernetes client)
         {
             if (node.Service == null) return;
             client.DeleteNamespacedService(node.Service.Name(), K8sNamespace);
@@ -191,7 +210,7 @@ namespace CodexDistTestCore
 
         #region Deployment management
 
-        private void CreateDeployment(ActiveNode node, Kubernetes client, OfflineCodexNode codexNode)
+        private void CreateDeployment(ActiveDeployment node, Kubernetes client, OfflineCodexNodes codexNode)
         {
             var deploymentSpec = new V1Deployment
             {
@@ -212,23 +231,7 @@ namespace CodexDistTestCore
                         },
                         Spec = new V1PodSpec
                         {
-                            Containers = new List<V1Container>
+                            Containers = CreateDeploymentContainers(node, codexNode)
-                            {
-                                new V1Container
-                                {
-                                    Name = node.GetContainerName(),
-                                    Image = dockerImage.GetImageTag(),
-                                    Ports = new List<V1ContainerPort>
-                                    {
-                                        new V1ContainerPort
-                                        {
-                                            ContainerPort = 8080,
-                                            Name = node.GetContainerPortName()
-                                        }
-                                    },
-                                    Env = dockerImage.CreateEnvironmentVariables(codexNode)
-                                }
-                            }
                         }
                     }
                 }
@@ -237,7 +240,30 @@ namespace CodexDistTestCore
             node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
         }
 
-        private void DeleteDeployment(ActiveNode node, Kubernetes client)
+        private List<V1Container> CreateDeploymentContainers(ActiveDeployment node,OfflineCodexNodes codexNode)
+        {
+            var result = new List<V1Container>();
+            foreach (var container in node.Containers)
+            {
+                result.Add(new V1Container
+                {
+                    Name = container.Name,
+                    Image = dockerImage.GetImageTag(),
+                    Ports = new List<V1ContainerPort>
+                    {
+                        new V1ContainerPort
+                        {
+                            ContainerPort = container.ApiPort,
+                            Name = container.ContainerPortName
+                        }
+                    },
+                    Env = dockerImage.CreateEnvironmentVariables(codexNode, container)
+                });
+            }
+            return result;
+        }
+
+        private void DeleteDeployment(ActiveDeployment node, Kubernetes client)
         {
             if (node.Deployment == null) return;
             client.DeleteNamespacedDeployment(node.Deployment.Name(), K8sNamespace);
@@ -286,11 +312,11 @@ namespace CodexDistTestCore
             return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
         }
 
-        private ActiveNode GetAndRemoveActiveNodeFor(IOnlineCodexNode node)
+        private ActiveDeployment GetAndRemoveActiveNodeFor(IOnlineCodexNodes node)
         {
-            var n = (OnlineCodexNode)node;
+            var n = (OnlineCodexNodes)node;
-            var activeNode = activeNodes[n];
+            var activeNode = activeDeployments[n];
-            activeNodes.Remove(n);
+            activeDeployments.Remove(n);
             return activeNode;
         }
     }

CodexDistTestCore/NumberSource.cs

@@ -2,27 +2,18 @@
 {
     public class NumberSource
     {
-        private int freePort;
+        private int number;
-        private int nodeOrderNumber;
 
-        public NumberSource()
+        public NumberSource(int start)
         {
-            freePort = 30001;
+            number = start;
-            nodeOrderNumber = 0;
         }
 
-        public int GetFreePort()
+        public int GetNextNumber()
         {
-            var port = freePort;
+            var n = number;
-            freePort++;
+            number++;
-            return port;
+            return n;
-        }
-
-        public int GetNodeOrderNumber()
-        {
-            var number = nodeOrderNumber;
-            nodeOrderNumber++;
-            return number;
         }
     }
 }

CodexDistTestCore/OfflineCodexNodes.cs

@@ -1,11 +1,11 @@
 namespace CodexDistTestCore
 {
-    public interface IOfflineCodexNode
+    public interface IOfflineCodexNodes
     {
-        IOfflineCodexNode WithLogLevel(CodexLogLevel level);
+        IOfflineCodexNodes WithLogLevel(CodexLogLevel level);
-        IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node);
+        IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node);
-        IOfflineCodexNode WithStorageQuota(ByteSize storageQuota);
+        IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota);
-        IOnlineCodexNode BringOnline();
+        IOnlineCodexNodes BringOnline();
     }
 
     public enum CodexLogLevel
@@ -17,37 +17,39 @@
         Error
     }
 
-    public class OfflineCodexNode : IOfflineCodexNode
+    public class OfflineCodexNodes : IOfflineCodexNodes
     {
         private readonly IK8sManager k8SManager;
-
+        
+        public int NumberOfNodes { get; }
         public CodexLogLevel? LogLevel { get; private set; }
         public IOnlineCodexNode? BootstrapNode { get; private set; }
         public ByteSize? StorageQuota { get; private set; }
 
-        public OfflineCodexNode(IK8sManager k8SManager)
+        public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes)
         {
             this.k8SManager = k8SManager;
+            NumberOfNodes = numberOfNodes;
         }
 
-        public IOnlineCodexNode BringOnline()
+        public IOnlineCodexNodes BringOnline()
         {
             return k8SManager.BringOnline(this);
         }
 
-        public IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node)
+        public IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node)
         {
             BootstrapNode = node;
             return this;
         }
 
-        public IOfflineCodexNode WithLogLevel(CodexLogLevel level)
+        public IOfflineCodexNodes WithLogLevel(CodexLogLevel level)
         {
             LogLevel = level;
             return this;
         }
 
-        public IOfflineCodexNode WithStorageQuota(ByteSize storageQuota)
+        public IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota)
         {
             StorageQuota = storageQuota;
             return this;

CodexDistTestCore/OnlineCodexNode.cs

@@ -7,25 +7,17 @@ namespace CodexDistTestCore
         CodexDebugResponse GetDebugInfo();
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId);
-        IOfflineCodexNode BringOffline();
     }
 
     public class OnlineCodexNode : IOnlineCodexNode
     {
-        private readonly IK8sManager k8SManager;
         private readonly IFileManager fileManager;
-        private readonly int port;
+        private readonly CodexNodeContainer environment;
 
-        public OnlineCodexNode(IK8sManager k8SManager, IFileManager fileManager, int port)
+        public OnlineCodexNode(IFileManager fileManager, CodexNodeContainer environment)
         {
-            this.k8SManager = k8SManager;
             this.fileManager = fileManager;
-            this.port = port;
+            this.environment = environment;
-        }
-
-        public IOfflineCodexNode BringOffline()
-        {
-            return k8SManager.BringOffline(this);
         }
 
         public CodexDebugResponse GetDebugInfo()
@@ -55,7 +47,7 @@ namespace CodexDistTestCore
 
         private Http Http()
         {
-            return new Http(ip: "127.0.0.1", port: port, baseUrl: "/api/codex/v1");
+            return new Http(ip: "127.0.0.1", port: environment.ServicePort, baseUrl: "/api/codex/v1");
         }
     }
 

CodexDistTestCore/OnlineCodexNodes.cs

@@ -0,0 +1,33 @@
+namespace CodexDistTestCore
+{
+    public interface IOnlineCodexNodes
+    {
+        IOfflineCodexNodes BringOffline();
+        IOnlineCodexNode this[int index] { get; }
+    }
+
+    public class OnlineCodexNodes : IOnlineCodexNodes
+    {
+        private readonly IK8sManager k8SManager;
+        private readonly IOnlineCodexNode[] nodes;
+
+        public OnlineCodexNodes(IK8sManager k8SManager, IOnlineCodexNode[] nodes)
+        {
+            this.k8SManager = k8SManager;
+            this.nodes = nodes;
+        }
+
+        public IOnlineCodexNode this[int index]
+        {
+            get
+            {
+                return nodes[index];
+            }
+        }
+
+        public IOfflineCodexNodes BringOffline()
+        {
+            return k8SManager.BringOffline(this);
+        }
+    }
+}

LongTests/BasicTests/SimpleTests.cs

@@ -9,10 +9,10 @@ namespace LongTests.BasicTests
         [Test, UseLongTimeouts]
         public void OneClientLargeFileTest()
         {
-            var primary = SetupCodexNode()
+            var primary = SetupCodexNodes(1)
                                 .WithLogLevel(CodexLogLevel.Warn)
                                 .WithStorageQuota(10.GB())
-                                .BringOnline();
+                                .BringOnline()[0];
 
             var testFile = GenerateTestFile(1.GB());
 

Tests/BasicTests/SimpleTests.cs

@@ -11,7 +11,7 @@ namespace Tests.BasicTests
         {
             var dockerImage = new CodexDockerImage();
 
-            var node = SetupCodexNode().BringOnline();
+            var node = SetupCodexNodes(1).BringOnline()[0];
 
             var debugInfo = node.GetDebugInfo();
 
@@ -22,10 +22,10 @@ namespace Tests.BasicTests
         [Test]
         public void OneClientTest()
         {
-            var primary = SetupCodexNode()
+            var primary = SetupCodexNodes(1)
                                 .WithLogLevel(CodexLogLevel.Trace)
                                 .WithStorageQuota(2.MB())
-                                .BringOnline();
+                                .BringOnline()[0];
 
             var testFile = GenerateTestFile(1.MB());