From 82e29d02c9639ff11f9d7fbb134aad115e8776e9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 21 Mar 2023 15:17:48 +0100 Subject: [PATCH] this works --- .../{ActiveNode.cs => ActiveDeployment.cs} | 24 +-- CodexDistTestCore/CodexDockerImage.cs | 11 +- CodexDistTestCore/CodexNodeContainer.cs | 45 ++++++ CodexDistTestCore/DistTest.cs | 4 +- CodexDistTestCore/K8sManager.cs | 140 +++++++++++------- CodexDistTestCore/NumberSource.cs | 23 +-- ...flineCodexNode.cs => OfflineCodexNodes.cs} | 26 ++-- CodexDistTestCore/OnlineCodexNode.cs | 16 +- CodexDistTestCore/OnlineCodexNodes.cs | 33 +++++ LongTests/BasicTests/SimpleTests.cs | 4 +- Tests/BasicTests/SimpleTests.cs | 6 +- 11 files changed, 207 insertions(+), 125 deletions(-) rename CodexDistTestCore/{ActiveNode.cs => ActiveDeployment.cs} (64%) create mode 100644 CodexDistTestCore/CodexNodeContainer.cs rename CodexDistTestCore/{OfflineCodexNode.cs => OfflineCodexNodes.cs} (59%) create mode 100644 CodexDistTestCore/OnlineCodexNodes.cs diff --git a/CodexDistTestCore/ActiveNode.cs b/CodexDistTestCore/ActiveDeployment.cs similarity index 64% rename from CodexDistTestCore/ActiveNode.cs rename to CodexDistTestCore/ActiveDeployment.cs index a5df3b7..f35642e 100644 --- a/CodexDistTestCore/ActiveNode.cs +++ b/CodexDistTestCore/ActiveDeployment.cs @@ -2,18 +2,18 @@ namespace CodexDistTestCore { - public class ActiveNode + public class ActiveDeployment { - public ActiveNode(OfflineCodexNode origin, int port, int orderNumber) + public ActiveDeployment(OfflineCodexNodes origin, int orderNumber, CodexNodeContainer[] containers) { Origin = origin; + Containers = containers; SelectorName = orderNumber.ToString().PadLeft(6, '0'); - Port = port; } - public OfflineCodexNode Origin { get; } + public OfflineCodexNodes Origin { get; } + public CodexNodeContainer[] Containers { get; } public string SelectorName { get; } - public int Port { get; } public V1Deployment? Deployment { get; set; } public V1Service? Service { get; set; } public List ActivePodNames { get; } = new List(); @@ -41,21 +41,9 @@ namespace CodexDistTestCore return new Dictionary { { "codex-test-node", "dist-test-" + SelectorName } }; } - public string GetContainerPortName() - { - //Caution, was: "codex-api-port" + SelectorName - //but string length causes 'UnprocessableEntity' exception in k8s. - return "api-" + SelectorName; - } - - public string GetContainerName() - { - return "codex-test-node"; - } - public string Describe() { - return $"CodexNode{SelectorName}-Port:{Port}-{Origin.Describe()}"; + return $"CodexNode{SelectorName}-{Origin.Describe()}"; } } } diff --git a/CodexDistTestCore/CodexDockerImage.cs b/CodexDistTestCore/CodexDockerImage.cs index 42b4137..a7dd353 100644 --- a/CodexDistTestCore/CodexDockerImage.cs +++ b/CodexDistTestCore/CodexDockerImage.cs @@ -14,10 +14,10 @@ namespace CodexDistTestCore return "b20483"; } - public List CreateEnvironmentVariables(OfflineCodexNode node) + public List CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer environment) { var formatter = new EnvFormatter(); - formatter.Create(node); + formatter.Create(node, environment); return formatter.Result; } @@ -25,8 +25,13 @@ namespace CodexDistTestCore { public List Result { get; } = new List(); - public void Create(OfflineCodexNode node) + public void Create(OfflineCodexNodes node, CodexNodeContainer environment) { + AddVar("API_PORT", environment.ApiPort.ToString()); + AddVar("DATA_DIR", environment.DataDir); + AddVar("DISC_PORT", environment.DiscoveryPort.ToString()); + AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{environment.ListenPort}"); + if (node.BootstrapNode != null) { var debugInfo = node.BootstrapNode.GetDebugInfo(); diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs new file mode 100644 index 0000000..0926ab8 --- /dev/null +++ b/CodexDistTestCore/CodexNodeContainer.cs @@ -0,0 +1,45 @@ +namespace CodexDistTestCore +{ + public class CodexNodeContainer + { + public CodexNodeContainer(string name, int servicePort, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir) + { + Name = name; + ServicePort = servicePort; + ApiPort = apiPort; + ContainerPortName = containerPortName; + DiscoveryPort = discoveryPort; + ListenPort = listenPort; + DataDir = dataDir; + } + + public string Name { get; } + public int ServicePort { get; } + public int ApiPort { get; } + public string ContainerPortName { get; } + public int DiscoveryPort { get; } + public int ListenPort { get; } + public string DataDir { get; } + } + + public class CodexNodeContainerFactory + { + private readonly NumberSource containerNameSource = new NumberSource(1); + private readonly NumberSource servicePortSource = new NumberSource(30001); + private readonly NumberSource codexPortSource = new NumberSource(8080); + + public CodexNodeContainer CreateNext() + { + var n = containerNameSource.GetNextNumber(); + return new CodexNodeContainer( + name: $"codex-node{n}", + servicePort: servicePortSource.GetNextNumber(), + apiPort: codexPortSource.GetNextNumber(), + containerPortName: $"api-{n}", + discoveryPort: codexPortSource.GetNextNumber(), + listenPort: codexPortSource.GetNextNumber(), + dataDir: $"datadir{n}" + ); + } + } +} diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index ff51d62..3e8ea98 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -56,9 +56,9 @@ namespace CodexDistTestCore return fileManager.GenerateTestFile(size); } - public IOfflineCodexNode SetupCodexNode() + public IOfflineCodexNodes SetupCodexNodes(int numberOfNodes) { - return new OfflineCodexNode(k8sManager); + return new OfflineCodexNodes(k8sManager, numberOfNodes); } } diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 5a61217..004782a 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -6,16 +6,16 @@ namespace CodexDistTestCore { public interface IK8sManager { - IOnlineCodexNode BringOnline(OfflineCodexNode node); - IOfflineCodexNode BringOffline(IOnlineCodexNode node); + IOnlineCodexNodes BringOnline(OfflineCodexNodes node); + IOfflineCodexNodes BringOffline(IOnlineCodexNodes node); } public class K8sManager : IK8sManager { public const string K8sNamespace = "codex-test-namespace"; private readonly CodexDockerImage dockerImage = new CodexDockerImage(); - private readonly NumberSource numberSource = new NumberSource(); - private readonly Dictionary activeNodes = new Dictionary(); + private readonly NumberSource activeDeploymentOrderNumberSource = new NumberSource(0); + private readonly Dictionary activeDeployments = new Dictionary(); private readonly List knownActivePodNames = new List(); private readonly IFileManager fileManager; @@ -24,26 +24,38 @@ namespace CodexDistTestCore this.fileManager = fileManager; } - public IOnlineCodexNode BringOnline(OfflineCodexNode node) + public IOnlineCodexNodes BringOnline(OfflineCodexNodes node) { var client = CreateClient(); - EnsureTestNamespace(client); - var activeNode = new ActiveNode(node, numberSource.GetFreePort(), numberSource.GetNodeOrderNumber()); - var codexNode = new OnlineCodexNode(this, fileManager, activeNode.Port); - activeNodes.Add(codexNode, activeNode); + var factory = new CodexNodeContainerFactory(); + var list = new List(); + var containers = new List(); + for (var i = 0; i < node.NumberOfNodes; i++) + { + var container = factory.CreateNext(); + containers.Add(container); - CreateDeployment(activeNode, client, node); - CreateService(activeNode, client); + var codexNode = new OnlineCodexNode(fileManager, container); + list.Add(codexNode); + } - WaitUntilOnline(activeNode, client); - TestLog.Log($"{activeNode.Describe()} online."); + var activeDeployment = new ActiveDeployment(node, activeDeploymentOrderNumberSource.GetNextNumber(), containers.ToArray()); - return codexNode; + var result = new OnlineCodexNodes(this, list.ToArray()); + activeDeployments.Add(result, activeDeployment); + + CreateDeployment(activeDeployment, client, node); + CreateService(activeDeployment, client); + + WaitUntilOnline(activeDeployment, client); + TestLog.Log($"{node.NumberOfNodes} Codex nodes online."); + + return result; } - public IOfflineCodexNode BringOffline(IOnlineCodexNode node) + public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node) { var client = CreateClient(); @@ -70,7 +82,7 @@ namespace CodexDistTestCore public void FetchAllPodsLogs(Action onLog) { var client = CreateClient(); - foreach (var node in activeNodes.Values) + foreach (var node in activeDeployments.Values) { var nodeDescription = node.Describe(); foreach (var podName in node.ActivePodNames) @@ -81,7 +93,7 @@ namespace CodexDistTestCore } } - private void BringOffline(ActiveNode activeNode, Kubernetes client) + private void BringOffline(ActiveDeployment activeNode, Kubernetes client) { DeleteDeployment(activeNode, client); DeleteService(activeNode, client); @@ -89,7 +101,7 @@ namespace CodexDistTestCore #region Waiting - private void WaitUntilOnline(ActiveNode activeNode, Kubernetes client) + private void WaitUntilOnline(ActiveDeployment activeNode, Kubernetes client) { WaitUntil(() => { @@ -100,7 +112,7 @@ namespace CodexDistTestCore AssignActivePodNames(activeNode, client); } - private void AssignActivePodNames(ActiveNode activeNode, Kubernetes client) + private void AssignActivePodNames(ActiveDeployment activeNode, Kubernetes client) { var pods = client.ListNamespacedPod(K8sNamespace); var podNames = pods.Items.Select(p => p.Name()); @@ -154,33 +166,40 @@ namespace CodexDistTestCore #region Service management - private void CreateService(ActiveNode node, Kubernetes client) + private void CreateService(ActiveDeployment activeDeployment, Kubernetes client) { var serviceSpec = new V1Service { ApiVersion = "v1", - Metadata = node.GetServiceMetadata(), + Metadata = activeDeployment.GetServiceMetadata(), Spec = new V1ServiceSpec { Type = "NodePort", - Selector = node.GetSelector(), - Ports = new List - { - new V1ServicePort - { - Protocol = "TCP", - Port = 8080, - TargetPort = node.GetContainerPortName(), - NodePort = node.Port - } - } + Selector = activeDeployment.GetSelector(), + Ports = CreateServicePorts(activeDeployment) } }; - node.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); + activeDeployment.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); } - private void DeleteService(ActiveNode node, Kubernetes client) + private List CreateServicePorts(ActiveDeployment activeDeployment) + { + var result = new List(); + foreach (var container in activeDeployment.Containers) + { + result.Add(new V1ServicePort + { + Protocol = "TCP", + Port = 8080, + TargetPort = container.ContainerPortName, + NodePort = container.ServicePort + }); + } + return result; + } + + private void DeleteService(ActiveDeployment node, Kubernetes client) { if (node.Service == null) return; client.DeleteNamespacedService(node.Service.Name(), K8sNamespace); @@ -191,7 +210,7 @@ namespace CodexDistTestCore #region Deployment management - private void CreateDeployment(ActiveNode node, Kubernetes client, OfflineCodexNode codexNode) + private void CreateDeployment(ActiveDeployment node, Kubernetes client, OfflineCodexNodes codexNode) { var deploymentSpec = new V1Deployment { @@ -212,23 +231,7 @@ namespace CodexDistTestCore }, Spec = new V1PodSpec { - Containers = new List - { - new V1Container - { - Name = node.GetContainerName(), - Image = dockerImage.GetImageTag(), - Ports = new List - { - new V1ContainerPort - { - ContainerPort = 8080, - Name = node.GetContainerPortName() - } - }, - Env = dockerImage.CreateEnvironmentVariables(codexNode) - } - } + Containers = CreateDeploymentContainers(node, codexNode) } } } @@ -237,7 +240,30 @@ namespace CodexDistTestCore node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); } - private void DeleteDeployment(ActiveNode node, Kubernetes client) + private List CreateDeploymentContainers(ActiveDeployment node,OfflineCodexNodes codexNode) + { + var result = new List(); + foreach (var container in node.Containers) + { + result.Add(new V1Container + { + Name = container.Name, + Image = dockerImage.GetImageTag(), + Ports = new List + { + new V1ContainerPort + { + ContainerPort = container.ApiPort, + Name = container.ContainerPortName + } + }, + Env = dockerImage.CreateEnvironmentVariables(codexNode, container) + }); + } + return result; + } + + private void DeleteDeployment(ActiveDeployment node, Kubernetes client) { if (node.Deployment == null) return; client.DeleteNamespacedDeployment(node.Deployment.Name(), K8sNamespace); @@ -286,11 +312,11 @@ namespace CodexDistTestCore return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); } - private ActiveNode GetAndRemoveActiveNodeFor(IOnlineCodexNode node) + private ActiveDeployment GetAndRemoveActiveNodeFor(IOnlineCodexNodes node) { - var n = (OnlineCodexNode)node; - var activeNode = activeNodes[n]; - activeNodes.Remove(n); + var n = (OnlineCodexNodes)node; + var activeNode = activeDeployments[n]; + activeDeployments.Remove(n); return activeNode; } } diff --git a/CodexDistTestCore/NumberSource.cs b/CodexDistTestCore/NumberSource.cs index 3dfa07b..4338610 100644 --- a/CodexDistTestCore/NumberSource.cs +++ b/CodexDistTestCore/NumberSource.cs @@ -2,27 +2,18 @@ { public class NumberSource { - private int freePort; - private int nodeOrderNumber; + private int number; - public NumberSource() + public NumberSource(int start) { - freePort = 30001; - nodeOrderNumber = 0; + number = start; } - public int GetFreePort() + public int GetNextNumber() { - var port = freePort; - freePort++; - return port; - } - - public int GetNodeOrderNumber() - { - var number = nodeOrderNumber; - nodeOrderNumber++; - return number; + var n = number; + number++; + return n; } } } diff --git a/CodexDistTestCore/OfflineCodexNode.cs b/CodexDistTestCore/OfflineCodexNodes.cs similarity index 59% rename from CodexDistTestCore/OfflineCodexNode.cs rename to CodexDistTestCore/OfflineCodexNodes.cs index 6eb018f..10e0093 100644 --- a/CodexDistTestCore/OfflineCodexNode.cs +++ b/CodexDistTestCore/OfflineCodexNodes.cs @@ -1,11 +1,11 @@ namespace CodexDistTestCore { - public interface IOfflineCodexNode + public interface IOfflineCodexNodes { - IOfflineCodexNode WithLogLevel(CodexLogLevel level); - IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node); - IOfflineCodexNode WithStorageQuota(ByteSize storageQuota); - IOnlineCodexNode BringOnline(); + IOfflineCodexNodes WithLogLevel(CodexLogLevel level); + IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node); + IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota); + IOnlineCodexNodes BringOnline(); } public enum CodexLogLevel @@ -17,37 +17,39 @@ Error } - public class OfflineCodexNode : IOfflineCodexNode + public class OfflineCodexNodes : IOfflineCodexNodes { private readonly IK8sManager k8SManager; - + + public int NumberOfNodes { get; } public CodexLogLevel? LogLevel { get; private set; } public IOnlineCodexNode? BootstrapNode { get; private set; } public ByteSize? StorageQuota { get; private set; } - public OfflineCodexNode(IK8sManager k8SManager) + public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes) { this.k8SManager = k8SManager; + NumberOfNodes = numberOfNodes; } - public IOnlineCodexNode BringOnline() + public IOnlineCodexNodes BringOnline() { return k8SManager.BringOnline(this); } - public IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node) + public IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node) { BootstrapNode = node; return this; } - public IOfflineCodexNode WithLogLevel(CodexLogLevel level) + public IOfflineCodexNodes WithLogLevel(CodexLogLevel level) { LogLevel = level; return this; } - public IOfflineCodexNode WithStorageQuota(ByteSize storageQuota) + public IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota) { StorageQuota = storageQuota; return this; diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index d6db1af..8acef06 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -7,25 +7,17 @@ namespace CodexDistTestCore CodexDebugResponse GetDebugInfo(); ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); - IOfflineCodexNode BringOffline(); } public class OnlineCodexNode : IOnlineCodexNode { - private readonly IK8sManager k8SManager; private readonly IFileManager fileManager; - private readonly int port; + private readonly CodexNodeContainer environment; - public OnlineCodexNode(IK8sManager k8SManager, IFileManager fileManager, int port) + public OnlineCodexNode(IFileManager fileManager, CodexNodeContainer environment) { - this.k8SManager = k8SManager; this.fileManager = fileManager; - this.port = port; - } - - public IOfflineCodexNode BringOffline() - { - return k8SManager.BringOffline(this); + this.environment = environment; } public CodexDebugResponse GetDebugInfo() @@ -55,7 +47,7 @@ namespace CodexDistTestCore private Http Http() { - return new Http(ip: "127.0.0.1", port: port, baseUrl: "/api/codex/v1"); + return new Http(ip: "127.0.0.1", port: environment.ServicePort, baseUrl: "/api/codex/v1"); } } diff --git a/CodexDistTestCore/OnlineCodexNodes.cs b/CodexDistTestCore/OnlineCodexNodes.cs new file mode 100644 index 0000000..0f0280a --- /dev/null +++ b/CodexDistTestCore/OnlineCodexNodes.cs @@ -0,0 +1,33 @@ +namespace CodexDistTestCore +{ + public interface IOnlineCodexNodes + { + IOfflineCodexNodes BringOffline(); + IOnlineCodexNode this[int index] { get; } + } + + public class OnlineCodexNodes : IOnlineCodexNodes + { + private readonly IK8sManager k8SManager; + private readonly IOnlineCodexNode[] nodes; + + public OnlineCodexNodes(IK8sManager k8SManager, IOnlineCodexNode[] nodes) + { + this.k8SManager = k8SManager; + this.nodes = nodes; + } + + public IOnlineCodexNode this[int index] + { + get + { + return nodes[index]; + } + } + + public IOfflineCodexNodes BringOffline() + { + return k8SManager.BringOffline(this); + } + } +} diff --git a/LongTests/BasicTests/SimpleTests.cs b/LongTests/BasicTests/SimpleTests.cs index 8c71653..02dfaf1 100644 --- a/LongTests/BasicTests/SimpleTests.cs +++ b/LongTests/BasicTests/SimpleTests.cs @@ -9,10 +9,10 @@ namespace LongTests.BasicTests [Test, UseLongTimeouts] public void OneClientLargeFileTest() { - var primary = SetupCodexNode() + var primary = SetupCodexNodes(1) .WithLogLevel(CodexLogLevel.Warn) .WithStorageQuota(10.GB()) - .BringOnline(); + .BringOnline()[0]; var testFile = GenerateTestFile(1.GB()); diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 2b7b729..02fcecf 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -11,7 +11,7 @@ namespace Tests.BasicTests { var dockerImage = new CodexDockerImage(); - var node = SetupCodexNode().BringOnline(); + var node = SetupCodexNodes(1).BringOnline()[0]; var debugInfo = node.GetDebugInfo(); @@ -22,10 +22,10 @@ namespace Tests.BasicTests [Test] public void OneClientTest() { - var primary = SetupCodexNode() + var primary = SetupCodexNodes(1) .WithLogLevel(CodexLogLevel.Trace) .WithStorageQuota(2.MB()) - .BringOnline(); + .BringOnline()[0]; var testFile = GenerateTestFile(1.MB());