From 82e29d02c9639ff11f9d7fbb134aad115e8776e9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 21 Mar 2023 15:17:48 +0100 Subject: [PATCH 01/12] this works --- .../{ActiveNode.cs => ActiveDeployment.cs} | 24 +-- CodexDistTestCore/CodexDockerImage.cs | 11 +- CodexDistTestCore/CodexNodeContainer.cs | 45 ++++++ CodexDistTestCore/DistTest.cs | 4 +- CodexDistTestCore/K8sManager.cs | 140 +++++++++++------- CodexDistTestCore/NumberSource.cs | 23 +-- ...flineCodexNode.cs => OfflineCodexNodes.cs} | 26 ++-- CodexDistTestCore/OnlineCodexNode.cs | 16 +- CodexDistTestCore/OnlineCodexNodes.cs | 33 +++++ LongTests/BasicTests/SimpleTests.cs | 4 +- Tests/BasicTests/SimpleTests.cs | 6 +- 11 files changed, 207 insertions(+), 125 deletions(-) rename CodexDistTestCore/{ActiveNode.cs => ActiveDeployment.cs} (64%) create mode 100644 CodexDistTestCore/CodexNodeContainer.cs rename CodexDistTestCore/{OfflineCodexNode.cs => OfflineCodexNodes.cs} (59%) create mode 100644 CodexDistTestCore/OnlineCodexNodes.cs diff --git a/CodexDistTestCore/ActiveNode.cs b/CodexDistTestCore/ActiveDeployment.cs similarity index 64% rename from CodexDistTestCore/ActiveNode.cs rename to CodexDistTestCore/ActiveDeployment.cs index a5df3b7..f35642e 100644 --- a/CodexDistTestCore/ActiveNode.cs +++ b/CodexDistTestCore/ActiveDeployment.cs @@ -2,18 +2,18 @@ namespace CodexDistTestCore { - public class ActiveNode + public class ActiveDeployment { - public ActiveNode(OfflineCodexNode origin, int port, int orderNumber) + public ActiveDeployment(OfflineCodexNodes origin, int orderNumber, CodexNodeContainer[] containers) { Origin = origin; + Containers = containers; SelectorName = orderNumber.ToString().PadLeft(6, '0'); - Port = port; } - public OfflineCodexNode Origin { get; } + public OfflineCodexNodes Origin { get; } + public CodexNodeContainer[] Containers { get; } public string SelectorName { get; } - public int Port { get; } public V1Deployment? Deployment { get; set; } public V1Service? Service { get; set; } public List ActivePodNames { get; } = new List(); @@ -41,21 +41,9 @@ namespace CodexDistTestCore return new Dictionary { { "codex-test-node", "dist-test-" + SelectorName } }; } - public string GetContainerPortName() - { - //Caution, was: "codex-api-port" + SelectorName - //but string length causes 'UnprocessableEntity' exception in k8s. - return "api-" + SelectorName; - } - - public string GetContainerName() - { - return "codex-test-node"; - } - public string Describe() { - return $"CodexNode{SelectorName}-Port:{Port}-{Origin.Describe()}"; + return $"CodexNode{SelectorName}-{Origin.Describe()}"; } } } diff --git a/CodexDistTestCore/CodexDockerImage.cs b/CodexDistTestCore/CodexDockerImage.cs index 42b4137..a7dd353 100644 --- a/CodexDistTestCore/CodexDockerImage.cs +++ b/CodexDistTestCore/CodexDockerImage.cs @@ -14,10 +14,10 @@ namespace CodexDistTestCore return "b20483"; } - public List CreateEnvironmentVariables(OfflineCodexNode node) + public List CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer environment) { var formatter = new EnvFormatter(); - formatter.Create(node); + formatter.Create(node, environment); return formatter.Result; } @@ -25,8 +25,13 @@ namespace CodexDistTestCore { public List Result { get; } = new List(); - public void Create(OfflineCodexNode node) + public void Create(OfflineCodexNodes node, CodexNodeContainer environment) { + AddVar("API_PORT", environment.ApiPort.ToString()); + AddVar("DATA_DIR", environment.DataDir); + AddVar("DISC_PORT", environment.DiscoveryPort.ToString()); + AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{environment.ListenPort}"); + if (node.BootstrapNode != null) { var debugInfo = node.BootstrapNode.GetDebugInfo(); diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs new file mode 100644 index 0000000..0926ab8 --- /dev/null +++ b/CodexDistTestCore/CodexNodeContainer.cs @@ -0,0 +1,45 @@ +namespace CodexDistTestCore +{ + public class CodexNodeContainer + { + public CodexNodeContainer(string name, int servicePort, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir) + { + Name = name; + ServicePort = servicePort; + ApiPort = apiPort; + ContainerPortName = containerPortName; + DiscoveryPort = discoveryPort; + ListenPort = listenPort; + DataDir = dataDir; + } + + public string Name { get; } + public int ServicePort { get; } + public int ApiPort { get; } + public string ContainerPortName { get; } + public int DiscoveryPort { get; } + public int ListenPort { get; } + public string DataDir { get; } + } + + public class CodexNodeContainerFactory + { + private readonly NumberSource containerNameSource = new NumberSource(1); + private readonly NumberSource servicePortSource = new NumberSource(30001); + private readonly NumberSource codexPortSource = new NumberSource(8080); + + public CodexNodeContainer CreateNext() + { + var n = containerNameSource.GetNextNumber(); + return new CodexNodeContainer( + name: $"codex-node{n}", + servicePort: servicePortSource.GetNextNumber(), + apiPort: codexPortSource.GetNextNumber(), + containerPortName: $"api-{n}", + discoveryPort: codexPortSource.GetNextNumber(), + listenPort: codexPortSource.GetNextNumber(), + dataDir: $"datadir{n}" + ); + } + } +} diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index ff51d62..3e8ea98 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -56,9 +56,9 @@ namespace CodexDistTestCore return fileManager.GenerateTestFile(size); } - public IOfflineCodexNode SetupCodexNode() + public IOfflineCodexNodes SetupCodexNodes(int numberOfNodes) { - return new OfflineCodexNode(k8sManager); + return new OfflineCodexNodes(k8sManager, numberOfNodes); } } diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 5a61217..004782a 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -6,16 +6,16 @@ namespace CodexDistTestCore { public interface IK8sManager { - IOnlineCodexNode BringOnline(OfflineCodexNode node); - IOfflineCodexNode BringOffline(IOnlineCodexNode node); + IOnlineCodexNodes BringOnline(OfflineCodexNodes node); + IOfflineCodexNodes BringOffline(IOnlineCodexNodes node); } public class K8sManager : IK8sManager { public const string K8sNamespace = "codex-test-namespace"; private readonly CodexDockerImage dockerImage = new CodexDockerImage(); - private readonly NumberSource numberSource = new NumberSource(); - private readonly Dictionary activeNodes = new Dictionary(); + private readonly NumberSource activeDeploymentOrderNumberSource = new NumberSource(0); + private readonly Dictionary activeDeployments = new Dictionary(); private readonly List knownActivePodNames = new List(); private readonly IFileManager fileManager; @@ -24,26 +24,38 @@ namespace CodexDistTestCore this.fileManager = fileManager; } - public IOnlineCodexNode BringOnline(OfflineCodexNode node) + public IOnlineCodexNodes BringOnline(OfflineCodexNodes node) { var client = CreateClient(); - EnsureTestNamespace(client); - var activeNode = new ActiveNode(node, numberSource.GetFreePort(), numberSource.GetNodeOrderNumber()); - var codexNode = new OnlineCodexNode(this, fileManager, activeNode.Port); - activeNodes.Add(codexNode, activeNode); + var factory = new CodexNodeContainerFactory(); + var list = new List(); + var containers = new List(); + for (var i = 0; i < node.NumberOfNodes; i++) + { + var container = factory.CreateNext(); + containers.Add(container); - CreateDeployment(activeNode, client, node); - CreateService(activeNode, client); + var codexNode = new OnlineCodexNode(fileManager, container); + list.Add(codexNode); + } - WaitUntilOnline(activeNode, client); - TestLog.Log($"{activeNode.Describe()} online."); + var activeDeployment = new ActiveDeployment(node, activeDeploymentOrderNumberSource.GetNextNumber(), containers.ToArray()); - return codexNode; + var result = new OnlineCodexNodes(this, list.ToArray()); + activeDeployments.Add(result, activeDeployment); + + CreateDeployment(activeDeployment, client, node); + CreateService(activeDeployment, client); + + WaitUntilOnline(activeDeployment, client); + TestLog.Log($"{node.NumberOfNodes} Codex nodes online."); + + return result; } - public IOfflineCodexNode BringOffline(IOnlineCodexNode node) + public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node) { var client = CreateClient(); @@ -70,7 +82,7 @@ namespace CodexDistTestCore public void FetchAllPodsLogs(Action onLog) { var client = CreateClient(); - foreach (var node in activeNodes.Values) + foreach (var node in activeDeployments.Values) { var nodeDescription = node.Describe(); foreach (var podName in node.ActivePodNames) @@ -81,7 +93,7 @@ namespace CodexDistTestCore } } - private void BringOffline(ActiveNode activeNode, Kubernetes client) + private void BringOffline(ActiveDeployment activeNode, Kubernetes client) { DeleteDeployment(activeNode, client); DeleteService(activeNode, client); @@ -89,7 +101,7 @@ namespace CodexDistTestCore #region Waiting - private void WaitUntilOnline(ActiveNode activeNode, Kubernetes client) + private void WaitUntilOnline(ActiveDeployment activeNode, Kubernetes client) { WaitUntil(() => { @@ -100,7 +112,7 @@ namespace CodexDistTestCore AssignActivePodNames(activeNode, client); } - private void AssignActivePodNames(ActiveNode activeNode, Kubernetes client) + private void AssignActivePodNames(ActiveDeployment activeNode, Kubernetes client) { var pods = client.ListNamespacedPod(K8sNamespace); var podNames = pods.Items.Select(p => p.Name()); @@ -154,33 +166,40 @@ namespace CodexDistTestCore #region Service management - private void CreateService(ActiveNode node, Kubernetes client) + private void CreateService(ActiveDeployment activeDeployment, Kubernetes client) { var serviceSpec = new V1Service { ApiVersion = "v1", - Metadata = node.GetServiceMetadata(), + Metadata = activeDeployment.GetServiceMetadata(), Spec = new V1ServiceSpec { Type = "NodePort", - Selector = node.GetSelector(), - Ports = new List - { - new V1ServicePort - { - Protocol = "TCP", - Port = 8080, - TargetPort = node.GetContainerPortName(), - NodePort = node.Port - } - } + Selector = activeDeployment.GetSelector(), + Ports = CreateServicePorts(activeDeployment) } }; - node.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); + activeDeployment.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); } - private void DeleteService(ActiveNode node, Kubernetes client) + private List CreateServicePorts(ActiveDeployment activeDeployment) + { + var result = new List(); + foreach (var container in activeDeployment.Containers) + { + result.Add(new V1ServicePort + { + Protocol = "TCP", + Port = 8080, + TargetPort = container.ContainerPortName, + NodePort = container.ServicePort + }); + } + return result; + } + + private void DeleteService(ActiveDeployment node, Kubernetes client) { if (node.Service == null) return; client.DeleteNamespacedService(node.Service.Name(), K8sNamespace); @@ -191,7 +210,7 @@ namespace CodexDistTestCore #region Deployment management - private void CreateDeployment(ActiveNode node, Kubernetes client, OfflineCodexNode codexNode) + private void CreateDeployment(ActiveDeployment node, Kubernetes client, OfflineCodexNodes codexNode) { var deploymentSpec = new V1Deployment { @@ -212,23 +231,7 @@ namespace CodexDistTestCore }, Spec = new V1PodSpec { - Containers = new List - { - new V1Container - { - Name = node.GetContainerName(), - Image = dockerImage.GetImageTag(), - Ports = new List - { - new V1ContainerPort - { - ContainerPort = 8080, - Name = node.GetContainerPortName() - } - }, - Env = dockerImage.CreateEnvironmentVariables(codexNode) - } - } + Containers = CreateDeploymentContainers(node, codexNode) } } } @@ -237,7 +240,30 @@ namespace CodexDistTestCore node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); } - private void DeleteDeployment(ActiveNode node, Kubernetes client) + private List CreateDeploymentContainers(ActiveDeployment node,OfflineCodexNodes codexNode) + { + var result = new List(); + foreach (var container in node.Containers) + { + result.Add(new V1Container + { + Name = container.Name, + Image = dockerImage.GetImageTag(), + Ports = new List + { + new V1ContainerPort + { + ContainerPort = container.ApiPort, + Name = container.ContainerPortName + } + }, + Env = dockerImage.CreateEnvironmentVariables(codexNode, container) + }); + } + return result; + } + + private void DeleteDeployment(ActiveDeployment node, Kubernetes client) { if (node.Deployment == null) return; client.DeleteNamespacedDeployment(node.Deployment.Name(), K8sNamespace); @@ -286,11 +312,11 @@ namespace CodexDistTestCore return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); } - private ActiveNode GetAndRemoveActiveNodeFor(IOnlineCodexNode node) + private ActiveDeployment GetAndRemoveActiveNodeFor(IOnlineCodexNodes node) { - var n = (OnlineCodexNode)node; - var activeNode = activeNodes[n]; - activeNodes.Remove(n); + var n = (OnlineCodexNodes)node; + var activeNode = activeDeployments[n]; + activeDeployments.Remove(n); return activeNode; } } diff --git a/CodexDistTestCore/NumberSource.cs b/CodexDistTestCore/NumberSource.cs index 3dfa07b..4338610 100644 --- a/CodexDistTestCore/NumberSource.cs +++ b/CodexDistTestCore/NumberSource.cs @@ -2,27 +2,18 @@ { public class NumberSource { - private int freePort; - private int nodeOrderNumber; + private int number; - public NumberSource() + public NumberSource(int start) { - freePort = 30001; - nodeOrderNumber = 0; + number = start; } - public int GetFreePort() + public int GetNextNumber() { - var port = freePort; - freePort++; - return port; - } - - public int GetNodeOrderNumber() - { - var number = nodeOrderNumber; - nodeOrderNumber++; - return number; + var n = number; + number++; + return n; } } } diff --git a/CodexDistTestCore/OfflineCodexNode.cs b/CodexDistTestCore/OfflineCodexNodes.cs similarity index 59% rename from CodexDistTestCore/OfflineCodexNode.cs rename to CodexDistTestCore/OfflineCodexNodes.cs index 6eb018f..10e0093 100644 --- a/CodexDistTestCore/OfflineCodexNode.cs +++ b/CodexDistTestCore/OfflineCodexNodes.cs @@ -1,11 +1,11 @@ namespace CodexDistTestCore { - public interface IOfflineCodexNode + public interface IOfflineCodexNodes { - IOfflineCodexNode WithLogLevel(CodexLogLevel level); - IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node); - IOfflineCodexNode WithStorageQuota(ByteSize storageQuota); - IOnlineCodexNode BringOnline(); + IOfflineCodexNodes WithLogLevel(CodexLogLevel level); + IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node); + IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota); + IOnlineCodexNodes BringOnline(); } public enum CodexLogLevel @@ -17,37 +17,39 @@ Error } - public class OfflineCodexNode : IOfflineCodexNode + public class OfflineCodexNodes : IOfflineCodexNodes { private readonly IK8sManager k8SManager; - + + public int NumberOfNodes { get; } public CodexLogLevel? LogLevel { get; private set; } public IOnlineCodexNode? BootstrapNode { get; private set; } public ByteSize? StorageQuota { get; private set; } - public OfflineCodexNode(IK8sManager k8SManager) + public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes) { this.k8SManager = k8SManager; + NumberOfNodes = numberOfNodes; } - public IOnlineCodexNode BringOnline() + public IOnlineCodexNodes BringOnline() { return k8SManager.BringOnline(this); } - public IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node) + public IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node) { BootstrapNode = node; return this; } - public IOfflineCodexNode WithLogLevel(CodexLogLevel level) + public IOfflineCodexNodes WithLogLevel(CodexLogLevel level) { LogLevel = level; return this; } - public IOfflineCodexNode WithStorageQuota(ByteSize storageQuota) + public IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota) { StorageQuota = storageQuota; return this; diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index d6db1af..8acef06 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -7,25 +7,17 @@ namespace CodexDistTestCore CodexDebugResponse GetDebugInfo(); ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); - IOfflineCodexNode BringOffline(); } public class OnlineCodexNode : IOnlineCodexNode { - private readonly IK8sManager k8SManager; private readonly IFileManager fileManager; - private readonly int port; + private readonly CodexNodeContainer environment; - public OnlineCodexNode(IK8sManager k8SManager, IFileManager fileManager, int port) + public OnlineCodexNode(IFileManager fileManager, CodexNodeContainer environment) { - this.k8SManager = k8SManager; this.fileManager = fileManager; - this.port = port; - } - - public IOfflineCodexNode BringOffline() - { - return k8SManager.BringOffline(this); + this.environment = environment; } public CodexDebugResponse GetDebugInfo() @@ -55,7 +47,7 @@ namespace CodexDistTestCore private Http Http() { - return new Http(ip: "127.0.0.1", port: port, baseUrl: "/api/codex/v1"); + return new Http(ip: "127.0.0.1", port: environment.ServicePort, baseUrl: "/api/codex/v1"); } } diff --git a/CodexDistTestCore/OnlineCodexNodes.cs b/CodexDistTestCore/OnlineCodexNodes.cs new file mode 100644 index 0000000..0f0280a --- /dev/null +++ b/CodexDistTestCore/OnlineCodexNodes.cs @@ -0,0 +1,33 @@ +namespace CodexDistTestCore +{ + public interface IOnlineCodexNodes + { + IOfflineCodexNodes BringOffline(); + IOnlineCodexNode this[int index] { get; } + } + + public class OnlineCodexNodes : IOnlineCodexNodes + { + private readonly IK8sManager k8SManager; + private readonly IOnlineCodexNode[] nodes; + + public OnlineCodexNodes(IK8sManager k8SManager, IOnlineCodexNode[] nodes) + { + this.k8SManager = k8SManager; + this.nodes = nodes; + } + + public IOnlineCodexNode this[int index] + { + get + { + return nodes[index]; + } + } + + public IOfflineCodexNodes BringOffline() + { + return k8SManager.BringOffline(this); + } + } +} diff --git a/LongTests/BasicTests/SimpleTests.cs b/LongTests/BasicTests/SimpleTests.cs index 8c71653..02dfaf1 100644 --- a/LongTests/BasicTests/SimpleTests.cs +++ b/LongTests/BasicTests/SimpleTests.cs @@ -9,10 +9,10 @@ namespace LongTests.BasicTests [Test, UseLongTimeouts] public void OneClientLargeFileTest() { - var primary = SetupCodexNode() + var primary = SetupCodexNodes(1) .WithLogLevel(CodexLogLevel.Warn) .WithStorageQuota(10.GB()) - .BringOnline(); + .BringOnline()[0]; var testFile = GenerateTestFile(1.GB()); diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 2b7b729..02fcecf 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -11,7 +11,7 @@ namespace Tests.BasicTests { var dockerImage = new CodexDockerImage(); - var node = SetupCodexNode().BringOnline(); + var node = SetupCodexNodes(1).BringOnline()[0]; var debugInfo = node.GetDebugInfo(); @@ -22,10 +22,10 @@ namespace Tests.BasicTests [Test] public void OneClientTest() { - var primary = SetupCodexNode() + var primary = SetupCodexNodes(1) .WithLogLevel(CodexLogLevel.Trace) .WithStorageQuota(2.MB()) - .BringOnline(); + .BringOnline()[0]; var testFile = GenerateTestFile(1.MB()); From f3a5ed397643ec75b02d198471045e78d1940b64 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 21 Mar 2023 15:44:21 +0100 Subject: [PATCH 02/12] cleanup --- CodexDistTestCore/ActiveDeployment.cs | 49 ----------- CodexDistTestCore/K8sManager.cs | 121 +++++++++++++------------- CodexDistTestCore/OnlineCodexNode.cs | 7 +- CodexDistTestCore/OnlineCodexNodes.cs | 53 +++++++++-- CodexDistTestCore/TestLog.cs | 9 +- 5 files changed, 116 insertions(+), 123 deletions(-) delete mode 100644 CodexDistTestCore/ActiveDeployment.cs diff --git a/CodexDistTestCore/ActiveDeployment.cs b/CodexDistTestCore/ActiveDeployment.cs deleted file mode 100644 index f35642e..0000000 --- a/CodexDistTestCore/ActiveDeployment.cs +++ /dev/null @@ -1,49 +0,0 @@ -using k8s.Models; - -namespace CodexDistTestCore -{ - public class ActiveDeployment - { - public ActiveDeployment(OfflineCodexNodes origin, int orderNumber, CodexNodeContainer[] containers) - { - Origin = origin; - Containers = containers; - SelectorName = orderNumber.ToString().PadLeft(6, '0'); - } - - public OfflineCodexNodes Origin { get; } - public CodexNodeContainer[] Containers { get; } - public string SelectorName { get; } - public V1Deployment? Deployment { get; set; } - public V1Service? Service { get; set; } - public List ActivePodNames { get; } = new List(); - - public V1ObjectMeta GetServiceMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-entrypoint-" + SelectorName, - NamespaceProperty = K8sManager.K8sNamespace - }; - } - - public V1ObjectMeta GetDeploymentMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-node-" + SelectorName, - NamespaceProperty = K8sManager.K8sNamespace - }; - } - - public Dictionary GetSelector() - { - return new Dictionary { { "codex-test-node", "dist-test-" + SelectorName } }; - } - - public string Describe() - { - return $"CodexNode{SelectorName}-{Origin.Describe()}"; - } - } -} diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 004782a..726d225 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -15,7 +15,7 @@ namespace CodexDistTestCore public const string K8sNamespace = "codex-test-namespace"; private readonly CodexDockerImage dockerImage = new CodexDockerImage(); private readonly NumberSource activeDeploymentOrderNumberSource = new NumberSource(0); - private readonly Dictionary activeDeployments = new Dictionary(); + private readonly List activeCodexNodes = new List(); private readonly List knownActivePodNames = new List(); private readonly IFileManager fileManager; @@ -24,37 +24,33 @@ namespace CodexDistTestCore this.fileManager = fileManager; } - public IOnlineCodexNodes BringOnline(OfflineCodexNodes node) + public IOnlineCodexNodes BringOnline(OfflineCodexNodes offline) { var client = CreateClient(); EnsureTestNamespace(client); - var factory = new CodexNodeContainerFactory(); - var list = new List(); - var containers = new List(); - for (var i = 0; i < node.NumberOfNodes; i++) - { - var container = factory.CreateNext(); - containers.Add(container); + var containers = CreateContainers(offline.NumberOfNodes); + var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray(); + var result = new OnlineCodexNodes(activeDeploymentOrderNumberSource.GetNextNumber(), offline, this, online); + activeCodexNodes.Add(result); - var codexNode = new OnlineCodexNode(fileManager, container); - list.Add(codexNode); - } + CreateDeployment(client, result, offline); + CreateService(result, client); - var activeDeployment = new ActiveDeployment(node, activeDeploymentOrderNumberSource.GetNextNumber(), containers.ToArray()); - - var result = new OnlineCodexNodes(this, list.ToArray()); - activeDeployments.Add(result, activeDeployment); - - CreateDeployment(activeDeployment, client, node); - CreateService(activeDeployment, client); - - WaitUntilOnline(activeDeployment, client); - TestLog.Log($"{node.NumberOfNodes} Codex nodes online."); + WaitUntilOnline(result, client); + TestLog.Log($"{offline.NumberOfNodes} Codex nodes online."); return result; } + private CodexNodeContainer[] CreateContainers(int number) + { + var factory = new CodexNodeContainerFactory(); + var containers = new List(); + for (var i = 0; i < number; i++) containers.Add(factory.CreateNext()); + return containers.ToArray(); + } + public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node) { var client = CreateClient(); @@ -79,40 +75,40 @@ namespace CodexDistTestCore WaitUntilNamespaceDeleted(client); } - public void FetchAllPodsLogs(Action onLog) + public void FetchAllPodsLogs(Action onLog) { var client = CreateClient(); - foreach (var node in activeDeployments.Values) + foreach (var node in activeCodexNodes) { var nodeDescription = node.Describe(); foreach (var podName in node.ActivePodNames) { var stream = client.ReadNamespacedPodLog(podName, K8sNamespace); - onLog(node.SelectorName, $"{nodeDescription}:{podName}", stream); + onLog(node.OrderNumber, $"{nodeDescription}:{podName}", stream); } } } - private void BringOffline(ActiveDeployment activeNode, Kubernetes client) + private void BringOffline(OnlineCodexNodes online, Kubernetes client) { - DeleteDeployment(activeNode, client); - DeleteService(activeNode, client); + DeleteDeployment(client, online); + DeleteService(client, online); } #region Waiting - private void WaitUntilOnline(ActiveDeployment activeNode, Kubernetes client) + private void WaitUntilOnline(OnlineCodexNodes online, Kubernetes client) { WaitUntil(() => { - activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), K8sNamespace); - return activeNode.Deployment?.Status.AvailableReplicas != null && activeNode.Deployment.Status.AvailableReplicas > 0; + online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); + return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; }); - AssignActivePodNames(activeNode, client); + AssignActivePodNames(online, client); } - private void AssignActivePodNames(ActiveDeployment activeNode, Kubernetes client) + private void AssignActivePodNames(OnlineCodexNodes online, Kubernetes client) { var pods = client.ListNamespacedPod(K8sNamespace); var podNames = pods.Items.Select(p => p.Name()); @@ -121,7 +117,7 @@ namespace CodexDistTestCore if (!knownActivePodNames.Contains(podName)) { knownActivePodNames.Add(podName); - activeNode.ActivePodNames.Add(podName); + online.ActivePodNames.Add(podName); } } } @@ -166,27 +162,28 @@ namespace CodexDistTestCore #region Service management - private void CreateService(ActiveDeployment activeDeployment, Kubernetes client) + private void CreateService(OnlineCodexNodes online, Kubernetes client) { var serviceSpec = new V1Service { ApiVersion = "v1", - Metadata = activeDeployment.GetServiceMetadata(), + Metadata = online.GetServiceMetadata(), Spec = new V1ServiceSpec { Type = "NodePort", - Selector = activeDeployment.GetSelector(), - Ports = CreateServicePorts(activeDeployment) + Selector = online.GetSelector(), + Ports = CreateServicePorts(online) } }; - activeDeployment.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); + online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); } - private List CreateServicePorts(ActiveDeployment activeDeployment) + private List CreateServicePorts(OnlineCodexNodes online) { var result = new List(); - foreach (var container in activeDeployment.Containers) + var containers = online.GetContainers(); + foreach (var container in containers) { result.Add(new V1ServicePort { @@ -199,51 +196,52 @@ namespace CodexDistTestCore return result; } - private void DeleteService(ActiveDeployment node, Kubernetes client) + private void DeleteService(Kubernetes client, OnlineCodexNodes online) { - if (node.Service == null) return; - client.DeleteNamespacedService(node.Service.Name(), K8sNamespace); - node.Service = null; + if (online.Service == null) return; + client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); + online.Service = null; } #endregion #region Deployment management - private void CreateDeployment(ActiveDeployment node, Kubernetes client, OfflineCodexNodes codexNode) + private void CreateDeployment(Kubernetes client, OnlineCodexNodes online, OfflineCodexNodes offline) { var deploymentSpec = new V1Deployment { ApiVersion = "apps/v1", - Metadata = node.GetDeploymentMetadata(), + Metadata = online.GetDeploymentMetadata(), Spec = new V1DeploymentSpec { Replicas = 1, Selector = new V1LabelSelector { - MatchLabels = node.GetSelector() + MatchLabels = online.GetSelector() }, Template = new V1PodTemplateSpec { Metadata = new V1ObjectMeta { - Labels = node.GetSelector() + Labels = online.GetSelector() }, Spec = new V1PodSpec { - Containers = CreateDeploymentContainers(node, codexNode) + Containers = CreateDeploymentContainers(online, offline) } } } }; - node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); + online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); } - private List CreateDeploymentContainers(ActiveDeployment node,OfflineCodexNodes codexNode) + private List CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline) { var result = new List(); - foreach (var container in node.Containers) + var containers = online.GetContainers(); + foreach (var container in containers) { result.Add(new V1Container { @@ -257,17 +255,17 @@ namespace CodexDistTestCore Name = container.ContainerPortName } }, - Env = dockerImage.CreateEnvironmentVariables(codexNode, container) + Env = dockerImage.CreateEnvironmentVariables(offline, container) }); } return result; } - private void DeleteDeployment(ActiveDeployment node, Kubernetes client) + private void DeleteDeployment(Kubernetes client, OnlineCodexNodes online) { - if (node.Deployment == null) return; - client.DeleteNamespacedDeployment(node.Deployment.Name(), K8sNamespace); - node.Deployment = null; + if (online.Deployment == null) return; + client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace); + online.Deployment = null; } #endregion @@ -312,12 +310,11 @@ namespace CodexDistTestCore return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); } - private ActiveDeployment GetAndRemoveActiveNodeFor(IOnlineCodexNodes node) + private OnlineCodexNodes GetAndRemoveActiveNodeFor(IOnlineCodexNodes node) { var n = (OnlineCodexNodes)node; - var activeNode = activeDeployments[n]; - activeDeployments.Remove(n); - return activeNode; + activeCodexNodes.Remove(n); + return n; } } } diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index 8acef06..54a1d88 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -12,14 +12,15 @@ namespace CodexDistTestCore public class OnlineCodexNode : IOnlineCodexNode { private readonly IFileManager fileManager; - private readonly CodexNodeContainer environment; public OnlineCodexNode(IFileManager fileManager, CodexNodeContainer environment) { this.fileManager = fileManager; - this.environment = environment; + Container = environment; } + public CodexNodeContainer Container { get; } + public CodexDebugResponse GetDebugInfo() { return Http().HttpGetJson("debug/info"); @@ -47,7 +48,7 @@ namespace CodexDistTestCore private Http Http() { - return new Http(ip: "127.0.0.1", port: environment.ServicePort, baseUrl: "/api/codex/v1"); + return new Http(ip: "127.0.0.1", port: Container.ServicePort, baseUrl: "/api/codex/v1"); } } diff --git a/CodexDistTestCore/OnlineCodexNodes.cs b/CodexDistTestCore/OnlineCodexNodes.cs index 0f0280a..119cf1f 100644 --- a/CodexDistTestCore/OnlineCodexNodes.cs +++ b/CodexDistTestCore/OnlineCodexNodes.cs @@ -1,4 +1,6 @@ -namespace CodexDistTestCore +using k8s.Models; + +namespace CodexDistTestCore { public interface IOnlineCodexNodes { @@ -9,25 +11,66 @@ public class OnlineCodexNodes : IOnlineCodexNodes { private readonly IK8sManager k8SManager; - private readonly IOnlineCodexNode[] nodes; - public OnlineCodexNodes(IK8sManager k8SManager, IOnlineCodexNode[] nodes) + public OnlineCodexNodes(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes) { + OrderNumber = orderNumber; + Origin = origin; this.k8SManager = k8SManager; - this.nodes = nodes; + Nodes = nodes; } public IOnlineCodexNode this[int index] { get { - return nodes[index]; + return Nodes[index]; } } + public int OrderNumber { get; } + public OfflineCodexNodes Origin { get; } + public OnlineCodexNode[] Nodes { get; } + public V1Deployment? Deployment { get; set; } + public V1Service? Service { get; set; } + public List ActivePodNames { get; } = new List(); + public IOfflineCodexNodes BringOffline() { return k8SManager.BringOffline(this); } + + public CodexNodeContainer[] GetContainers() + { + return Nodes.Select(n => n.Container).ToArray(); + } + + public V1ObjectMeta GetServiceMetadata() + { + return new V1ObjectMeta + { + Name = "codex-test-entrypoint-" + OrderNumber, + NamespaceProperty = K8sManager.K8sNamespace + }; + } + + public V1ObjectMeta GetDeploymentMetadata() + { + return new V1ObjectMeta + { + Name = "codex-test-node-" + OrderNumber, + NamespaceProperty = K8sManager.K8sNamespace + }; + } + + public Dictionary GetSelector() + { + return new Dictionary { { "codex-test-node", "dist-test-" + OrderNumber } }; + } + + public string Describe() + { + return $"CodexNode{OrderNumber}-{Origin.Describe()}"; + } } } diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index 3854cd1..e376204 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -62,15 +62,16 @@ namespace CodexDistTestCore k8sManager.FetchAllPodsLogs(WritePodLog); } - private static void WritePodLog(string id, string nodeDescription, Stream stream) + private static void WritePodLog(int id, string nodeDescription, Stream stream) { - Log($"{nodeDescription} -->> {id}"); - LogRaw(nodeDescription, id); + var logFile = id.ToString().PadLeft(6, '0'); + Log($"{nodeDescription} -->> {logFile}"); + LogRaw(nodeDescription, logFile); var reader = new StreamReader(stream); var line = reader.ReadLine(); while (line != null) { - LogRaw(line, id); + LogRaw(line, logFile); line = reader.ReadLine(); } } From a36e364996b7fb09d7e2f21cccfe29ad9e85e180 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 21 Mar 2023 16:09:41 +0100 Subject: [PATCH 03/12] Cleanup of k8s manager --- CodexDistTestCore/K8sManager.cs | 322 ++++---------------------- CodexDistTestCore/K8sOperations.cs | 270 +++++++++++++++++++++ CodexDistTestCore/KnownK8sPods.cs | 17 ++ CodexDistTestCore/OnlineCodexNodes.cs | 14 +- CodexDistTestCore/PodLogsHandler.cs | 7 + CodexDistTestCore/TestLog.cs | 32 ++- 6 files changed, 365 insertions(+), 297 deletions(-) create mode 100644 CodexDistTestCore/K8sOperations.cs create mode 100644 CodexDistTestCore/KnownK8sPods.cs create mode 100644 CodexDistTestCore/PodLogsHandler.cs diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 726d225..45c6b42 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -1,8 +1,4 @@ -using k8s; -using k8s.Models; -using NUnit.Framework; - -namespace CodexDistTestCore +namespace CodexDistTestCore { public interface IK8sManager { @@ -12,11 +8,9 @@ namespace CodexDistTestCore public class K8sManager : IK8sManager { - public const string K8sNamespace = "codex-test-namespace"; - private readonly CodexDockerImage dockerImage = new CodexDockerImage(); - private readonly NumberSource activeDeploymentOrderNumberSource = new NumberSource(0); - private readonly List activeCodexNodes = new List(); - private readonly List knownActivePodNames = new List(); + private readonly NumberSource onlineCodexNodeOrderNumberSource = new NumberSource(0); + private readonly List onlineCodexNodes = new List(); + private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly IFileManager fileManager; public K8sManager(IFileManager fileManager) @@ -26,20 +20,42 @@ namespace CodexDistTestCore public IOnlineCodexNodes BringOnline(OfflineCodexNodes offline) { - var client = CreateClient(); - EnsureTestNamespace(client); + var online = CreateOnlineCodexNodes(offline); - var containers = CreateContainers(offline.NumberOfNodes); - var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray(); - var result = new OnlineCodexNodes(activeDeploymentOrderNumberSource.GetNextNumber(), offline, this, online); - activeCodexNodes.Add(result); + K8s().BringOnline(online, offline); - CreateDeployment(client, result, offline); - CreateService(result, client); - - WaitUntilOnline(result, client); TestLog.Log($"{offline.NumberOfNodes} Codex nodes online."); + return online; + } + + public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node) + { + var online = GetAndRemoveActiveNodeFor(node); + + K8s().BringOffline(online); + + TestLog.Log($"{online.Describe()} offline."); + + return online.Origin; + } + + public void DeleteAllResources() + { + K8s().DeleteAllResources(); + } + + public void FetchAllPodsLogs(IPodLogsHandler logHandler) + { + K8s().FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler); + } + + private OnlineCodexNodes CreateOnlineCodexNodes(OfflineCodexNodes offline) + { + var containers = CreateContainers(offline.NumberOfNodes); + var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray(); + var result = new OnlineCodexNodes(onlineCodexNodeOrderNumberSource.GetNextNumber(), offline, this, online); + onlineCodexNodes.Add(result); return result; } @@ -51,270 +67,16 @@ namespace CodexDistTestCore return containers.ToArray(); } - public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node) - { - var client = CreateClient(); - - var activeNode = GetAndRemoveActiveNodeFor(node); - - var deploymentName = activeNode.Deployment.Name(); - BringOffline(activeNode, client); - WaitUntilOffline(deploymentName, client); - TestLog.Log($"{activeNode.Describe()} offline."); - - return activeNode.Origin; - } - - public void DeleteAllResources() - { - var client = CreateClient(); - - DeleteNamespace(client); - - WaitUntilZeroPods(client); - WaitUntilNamespaceDeleted(client); - } - - public void FetchAllPodsLogs(Action onLog) - { - var client = CreateClient(); - foreach (var node in activeCodexNodes) - { - var nodeDescription = node.Describe(); - foreach (var podName in node.ActivePodNames) - { - var stream = client.ReadNamespacedPodLog(podName, K8sNamespace); - onLog(node.OrderNumber, $"{nodeDescription}:{podName}", stream); - } - } - } - - private void BringOffline(OnlineCodexNodes online, Kubernetes client) - { - DeleteDeployment(client, online); - DeleteService(client, online); - } - - #region Waiting - - private void WaitUntilOnline(OnlineCodexNodes online, Kubernetes client) - { - WaitUntil(() => - { - online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); - return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; - }); - - AssignActivePodNames(online, client); - } - - private void AssignActivePodNames(OnlineCodexNodes online, Kubernetes client) - { - var pods = client.ListNamespacedPod(K8sNamespace); - var podNames = pods.Items.Select(p => p.Name()); - foreach (var podName in podNames) - { - if (!knownActivePodNames.Contains(podName)) - { - knownActivePodNames.Add(podName); - online.ActivePodNames.Add(podName); - } - } - } - - private void WaitUntilOffline(string deploymentName, Kubernetes client) - { - WaitUntil(() => - { - var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); - return deployment == null || deployment.Status.AvailableReplicas == 0; - }); - } - - private void WaitUntilZeroPods(Kubernetes client) - { - WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); - } - - private void WaitUntilNamespaceDeleted(Kubernetes client) - { - WaitUntil(() => !IsTestNamespaceOnline(client)); - } - - private void WaitUntil(Func predicate) - { - var start = DateTime.UtcNow; - var state = predicate(); - while (!state) - { - if (DateTime.UtcNow - start > Timing.K8sOperationTimeout()) - { - Assert.Fail("K8s operation timed out."); - throw new TimeoutException(); - } - - Timing.WaitForK8sServiceDelay(); - state = predicate(); - } - } - - #endregion - - #region Service management - - private void CreateService(OnlineCodexNodes online, Kubernetes client) - { - var serviceSpec = new V1Service - { - ApiVersion = "v1", - Metadata = online.GetServiceMetadata(), - Spec = new V1ServiceSpec - { - Type = "NodePort", - Selector = online.GetSelector(), - Ports = CreateServicePorts(online) - } - }; - - online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); - } - - private List CreateServicePorts(OnlineCodexNodes online) - { - var result = new List(); - var containers = online.GetContainers(); - foreach (var container in containers) - { - result.Add(new V1ServicePort - { - Protocol = "TCP", - Port = 8080, - TargetPort = container.ContainerPortName, - NodePort = container.ServicePort - }); - } - return result; - } - - private void DeleteService(Kubernetes client, OnlineCodexNodes online) - { - if (online.Service == null) return; - client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); - online.Service = null; - } - - #endregion - - #region Deployment management - - private void CreateDeployment(Kubernetes client, OnlineCodexNodes online, OfflineCodexNodes offline) - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = online.GetDeploymentMetadata(), - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = online.GetSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = online.GetSelector() - }, - Spec = new V1PodSpec - { - Containers = CreateDeploymentContainers(online, offline) - } - } - } - }; - - online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); - } - - private List CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline) - { - var result = new List(); - var containers = online.GetContainers(); - foreach (var container in containers) - { - result.Add(new V1Container - { - Name = container.Name, - Image = dockerImage.GetImageTag(), - Ports = new List - { - new V1ContainerPort - { - ContainerPort = container.ApiPort, - Name = container.ContainerPortName - } - }, - Env = dockerImage.CreateEnvironmentVariables(offline, container) - }); - } - return result; - } - - private void DeleteDeployment(Kubernetes client, OnlineCodexNodes online) - { - if (online.Deployment == null) return; - client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace); - online.Deployment = null; - } - - #endregion - - #region Namespace management - - private void EnsureTestNamespace(Kubernetes client) - { - if (IsTestNamespaceOnline(client)) return; - - var namespaceSpec = new V1Namespace - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = K8sNamespace, - Labels = new Dictionary { { "name", K8sNamespace } } - } - }; - client.CreateNamespace(namespaceSpec); - } - - private void DeleteNamespace(Kubernetes client) - { - if (IsTestNamespaceOnline(client)) - { - client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0); - } - } - - #endregion - - private static Kubernetes CreateClient() - { - // todo: If the default KubeConfig file does not suffice, change it here: - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(); - return new Kubernetes(config); - } - - private static bool IsTestNamespaceOnline(Kubernetes client) - { - return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); - } - private OnlineCodexNodes GetAndRemoveActiveNodeFor(IOnlineCodexNodes node) { var n = (OnlineCodexNodes)node; - activeCodexNodes.Remove(n); + onlineCodexNodes.Remove(n); return n; } + + private K8sOperations K8s() + { + return new K8sOperations(knownPods); + } } } diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs new file mode 100644 index 0000000..38a10d4 --- /dev/null +++ b/CodexDistTestCore/K8sOperations.cs @@ -0,0 +1,270 @@ +using k8s; +using k8s.Models; +using NUnit.Framework; + +namespace CodexDistTestCore +{ + public class K8sOperations + { + public const string K8sNamespace = "codex-test-namespace"; + + private readonly CodexDockerImage dockerImage = new CodexDockerImage(); + private readonly Kubernetes client; + private readonly KnownK8sPods knownPods; + + public K8sOperations(KnownK8sPods knownPods) + { + this.knownPods = knownPods; + + // todo: If the default KubeConfig file does not suffice, change it here: + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(); + client = new Kubernetes(config); + } + + public void BringOnline(OnlineCodexNodes online, OfflineCodexNodes offline) + { + EnsureTestNamespace(); + + CreateDeployment(online, offline); + CreateService(online); + + WaitUntilOnline(online); + AssignActivePodNames(online); + } + + public void BringOffline(OnlineCodexNodes online) + { + var deploymentName = online.Deployment.Name(); + DeleteDeployment(online); + DeleteService(online); + WaitUntilOffline(deploymentName); + } + + public void DeleteAllResources() + { + DeleteNamespace(); + + WaitUntilZeroPods(); + WaitUntilNamespaceDeleted(); + } + + public void FetchAllPodsLogs(OnlineCodexNodes[] onlines, IPodLogsHandler logHandler) + { + foreach (var online in onlines) + { + var nodeDescription = online.Describe(); + foreach (var podName in online.ActivePodNames) + { + var stream = client.ReadNamespacedPodLog(podName, K8sNamespace); + logHandler.Log(online.OrderNumber, $"{nodeDescription}:{podName}", stream); + } + } + } + + private void AssignActivePodNames(OnlineCodexNodes online) + { + var pods = client.ListNamespacedPod(K8sNamespace); + var podNames = pods.Items.Select(p => p.Name()); + foreach (var podName in podNames) + { + if (!knownPods.Contains(podName)) + { + knownPods.Add(podName); + online.ActivePodNames.Add(podName); + } + } + } + + #region Waiting + + private void WaitUntilOnline(OnlineCodexNodes online) + { + WaitUntil(() => + { + online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); + return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; + }); + } + + private void WaitUntilOffline(string deploymentName) + { + WaitUntil(() => + { + var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); + return deployment == null || deployment.Status.AvailableReplicas == 0; + }); + } + + private void WaitUntilZeroPods() + { + WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); + } + + private void WaitUntilNamespaceDeleted() + { + WaitUntil(() => !IsTestNamespaceOnline()); + } + + private void WaitUntil(Func predicate) + { + var start = DateTime.UtcNow; + var state = predicate(); + while (!state) + { + if (DateTime.UtcNow - start > Timing.K8sOperationTimeout()) + { + Assert.Fail("K8s operation timed out."); + throw new TimeoutException(); + } + + Timing.WaitForK8sServiceDelay(); + state = predicate(); + } + } + + #endregion + + #region Service management + + private void CreateService(OnlineCodexNodes online) + { + var serviceSpec = new V1Service + { + ApiVersion = "v1", + Metadata = online.GetServiceMetadata(), + Spec = new V1ServiceSpec + { + Type = "NodePort", + Selector = online.GetSelector(), + Ports = CreateServicePorts(online) + } + }; + + online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); + } + + private List CreateServicePorts(OnlineCodexNodes online) + { + var result = new List(); + var containers = online.GetContainers(); + foreach (var container in containers) + { + result.Add(new V1ServicePort + { + Protocol = "TCP", + Port = 8080, + TargetPort = container.ContainerPortName, + NodePort = container.ServicePort + }); + } + return result; + } + + private void DeleteService(OnlineCodexNodes online) + { + if (online.Service == null) return; + client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); + online.Service = null; + } + + #endregion + + #region Deployment management + + private void CreateDeployment(OnlineCodexNodes online, OfflineCodexNodes offline) + { + var deploymentSpec = new V1Deployment + { + ApiVersion = "apps/v1", + Metadata = online.GetDeploymentMetadata(), + Spec = new V1DeploymentSpec + { + Replicas = 1, + Selector = new V1LabelSelector + { + MatchLabels = online.GetSelector() + }, + Template = new V1PodTemplateSpec + { + Metadata = new V1ObjectMeta + { + Labels = online.GetSelector() + }, + Spec = new V1PodSpec + { + Containers = CreateDeploymentContainers(online, offline) + } + } + } + }; + + online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); + } + + private List CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline) + { + var result = new List(); + var containers = online.GetContainers(); + foreach (var container in containers) + { + result.Add(new V1Container + { + Name = container.Name, + Image = dockerImage.GetImageTag(), + Ports = new List + { + new V1ContainerPort + { + ContainerPort = container.ApiPort, + Name = container.ContainerPortName + } + }, + Env = dockerImage.CreateEnvironmentVariables(offline, container) + }); + } + return result; + } + + private void DeleteDeployment(OnlineCodexNodes online) + { + if (online.Deployment == null) return; + client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace); + online.Deployment = null; + } + + #endregion + + #region Namespace management + + private void EnsureTestNamespace() + { + if (IsTestNamespaceOnline()) return; + + var namespaceSpec = new V1Namespace + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = K8sNamespace, + Labels = new Dictionary { { "name", K8sNamespace } } + } + }; + client.CreateNamespace(namespaceSpec); + } + + private void DeleteNamespace() + { + if (IsTestNamespaceOnline()) + { + client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0); + } + } + + #endregion + + private bool IsTestNamespaceOnline() + { + return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); + } + } +} diff --git a/CodexDistTestCore/KnownK8sPods.cs b/CodexDistTestCore/KnownK8sPods.cs new file mode 100644 index 0000000..940a147 --- /dev/null +++ b/CodexDistTestCore/KnownK8sPods.cs @@ -0,0 +1,17 @@ +namespace CodexDistTestCore +{ + public class KnownK8sPods + { + private readonly List knownActivePodNames = new List(); + + public bool Contains(string name) + { + return knownActivePodNames.Contains(name); + } + + public void Add(string name) + { + knownActivePodNames.Add(name); + } + } +} diff --git a/CodexDistTestCore/OnlineCodexNodes.cs b/CodexDistTestCore/OnlineCodexNodes.cs index 119cf1f..c4007fa 100644 --- a/CodexDistTestCore/OnlineCodexNodes.cs +++ b/CodexDistTestCore/OnlineCodexNodes.cs @@ -28,6 +28,11 @@ namespace CodexDistTestCore } } + public IOfflineCodexNodes BringOffline() + { + return k8SManager.BringOffline(this); + } + public int OrderNumber { get; } public OfflineCodexNodes Origin { get; } public OnlineCodexNode[] Nodes { get; } @@ -35,11 +40,6 @@ namespace CodexDistTestCore public V1Service? Service { get; set; } public List ActivePodNames { get; } = new List(); - public IOfflineCodexNodes BringOffline() - { - return k8SManager.BringOffline(this); - } - public CodexNodeContainer[] GetContainers() { return Nodes.Select(n => n.Container).ToArray(); @@ -50,7 +50,7 @@ namespace CodexDistTestCore return new V1ObjectMeta { Name = "codex-test-entrypoint-" + OrderNumber, - NamespaceProperty = K8sManager.K8sNamespace + NamespaceProperty = K8sOperations.K8sNamespace }; } @@ -59,7 +59,7 @@ namespace CodexDistTestCore return new V1ObjectMeta { Name = "codex-test-node-" + OrderNumber, - NamespaceProperty = K8sManager.K8sNamespace + NamespaceProperty = K8sOperations.K8sNamespace }; } diff --git a/CodexDistTestCore/PodLogsHandler.cs b/CodexDistTestCore/PodLogsHandler.cs new file mode 100644 index 0000000..d7898f5 --- /dev/null +++ b/CodexDistTestCore/PodLogsHandler.cs @@ -0,0 +1,7 @@ +namespace CodexDistTestCore +{ + public interface IPodLogsHandler + { + void Log(int id, string podDescription, Stream log); + } +} diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index e376204..04cad1c 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -8,6 +8,7 @@ namespace CodexDistTestCore private static LogFile? file = null; + // This is all way too static. It needs to be cleaned up. public static void Log(string message) { file!.Write(message); @@ -38,7 +39,8 @@ namespace CodexDistTestCore Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) { - IncludeFullPodLogging(k8sManager); + var logWriter = new PodLogWriter(file); + logWriter.IncludeFullPodLogging(k8sManager); } file = null; @@ -50,24 +52,29 @@ namespace CodexDistTestCore var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); return $"{className}.{test.MethodName}"; } + } - private static void LogRaw(string message, string filename) + public class PodLogWriter : IPodLogsHandler + { + private readonly LogFile file; + + public PodLogWriter(LogFile file) { - file!.WriteRaw(message, filename); + this.file = file; } - private static void IncludeFullPodLogging(K8sManager k8sManager) + public void IncludeFullPodLogging(K8sManager k8sManager) { - Log("Full pod logging:"); - k8sManager.FetchAllPodsLogs(WritePodLog); + TestLog.Log("Full pod logging:"); + k8sManager.FetchAllPodsLogs(this); } - private static void WritePodLog(int id, string nodeDescription, Stream stream) + public void Log(int id, string podDescription, Stream log) { var logFile = id.ToString().PadLeft(6, '0'); - Log($"{nodeDescription} -->> {logFile}"); - LogRaw(nodeDescription, logFile); - var reader = new StreamReader(stream); + TestLog.Log($"{podDescription} -->> {logFile}"); + LogRaw(podDescription, logFile); + var reader = new StreamReader(log); var line = reader.ReadLine(); while (line != null) { @@ -75,6 +82,11 @@ namespace CodexDistTestCore line = reader.ReadLine(); } } + + private void LogRaw(string message, string filename) + { + file!.WriteRaw(message, filename); + } } public class LogFile From a60124239c7a50af6f8eecef28c3b5e050faee65 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 22 Mar 2023 09:22:18 +0100 Subject: [PATCH 04/12] Better logging --- ...{OnlineCodexNodes.cs => CodexNodeGroup.cs} | 8 ++--- CodexDistTestCore/DistTest.cs | 30 +++++++++++----- CodexDistTestCore/FileManager.cs | 16 +++++++-- CodexDistTestCore/K8sManager.cs | 28 ++++++++------- CodexDistTestCore/K8sOperations.cs | 22 ++++++------ CodexDistTestCore/OfflineCodexNodes.cs | 18 ++++++---- CodexDistTestCore/OnlineCodexNode.cs | 19 ++++++++-- CodexDistTestCore/TestLog.cs | 35 +++++++------------ 8 files changed, 104 insertions(+), 72 deletions(-) rename CodexDistTestCore/{OnlineCodexNodes.cs => CodexNodeGroup.cs} (86%) diff --git a/CodexDistTestCore/OnlineCodexNodes.cs b/CodexDistTestCore/CodexNodeGroup.cs similarity index 86% rename from CodexDistTestCore/OnlineCodexNodes.cs rename to CodexDistTestCore/CodexNodeGroup.cs index c4007fa..23189e5 100644 --- a/CodexDistTestCore/OnlineCodexNodes.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -2,17 +2,17 @@ namespace CodexDistTestCore { - public interface IOnlineCodexNodes + public interface ICodexNodeGroup { IOfflineCodexNodes BringOffline(); IOnlineCodexNode this[int index] { get; } } - public class OnlineCodexNodes : IOnlineCodexNodes + public class CodexNodeGroup : ICodexNodeGroup { private readonly IK8sManager k8SManager; - public OnlineCodexNodes(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes) + public CodexNodeGroup(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes) { OrderNumber = orderNumber; Origin = origin; @@ -70,7 +70,7 @@ namespace CodexDistTestCore public string Describe() { - return $"CodexNode{OrderNumber}-{Origin.Describe()}"; + return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}"; } } } diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index 3e8ea98..55bd75c 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -5,6 +5,7 @@ namespace CodexDistTestCore [SetUpFixture] public abstract class DistTest { + private TestLog log = null!; private FileManager fileManager = null!; private K8sManager k8sManager = null!; @@ -13,11 +14,22 @@ namespace CodexDistTestCore { // Previous test run may have been interrupted. // Begin by cleaning everything up. - fileManager = new FileManager(); - k8sManager = new K8sManager(fileManager); + log = new TestLog(); + fileManager = new FileManager(log); + k8sManager = new K8sManager(log, fileManager); - k8sManager.DeleteAllResources(); - fileManager.DeleteAllTestFiles(); + try + { + k8sManager.DeleteAllResources(); + fileManager.DeleteAllTestFiles(); + } + catch (Exception ex) + { + GlobalTestFailure.HasFailed = true; + log.Error($"Global setup cleanup failed with: {ex}"); + throw; + } + log.Log("Global setup cleanup successful"); } [SetUp] @@ -29,9 +41,9 @@ namespace CodexDistTestCore } else { - TestLog.BeginTest(); - fileManager = new FileManager(); - k8sManager = new K8sManager(fileManager); + log = new TestLog(); + fileManager = new FileManager(log); + k8sManager = new K8sManager(log, fileManager); } } @@ -40,13 +52,13 @@ namespace CodexDistTestCore { try { - TestLog.EndTest(k8sManager); + log.EndTest(k8sManager); k8sManager.DeleteAllResources(); fileManager.DeleteAllTestFiles(); } catch (Exception ex) { - TestLog.Error("Cleanup failed: " + ex.Message); + log.Error("Cleanup failed: " + ex.Message); GlobalTestFailure.HasFailed = true; } } diff --git a/CodexDistTestCore/FileManager.cs b/CodexDistTestCore/FileManager.cs index a040e09..94abe7a 100644 --- a/CodexDistTestCore/FileManager.cs +++ b/CodexDistTestCore/FileManager.cs @@ -15,10 +15,12 @@ namespace CodexDistTestCore private const string Folder = "TestDataFiles"; private readonly Random random = new Random(); private readonly List activeFiles = new List(); + private readonly TestLog log; - public FileManager() + public FileManager(TestLog log) { if (!Directory.Exists(Folder)) Directory.CreateDirectory(Folder); + this.log = log; } public TestFile CreateEmptyTestFile() @@ -26,7 +28,7 @@ namespace CodexDistTestCore var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin")); File.Create(result.Filename).Close(); activeFiles.Add(result); - TestLog.Log($"Created test file '{result.Filename}'."); + log.Log($"Created test file '{result.Filename}'."); return result; } @@ -34,7 +36,7 @@ namespace CodexDistTestCore { var result = CreateEmptyTestFile(); GenerateFileBytes(result, size); - TestLog.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'."); + log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'."); return result; } @@ -73,11 +75,19 @@ namespace CodexDistTestCore public string Filename { get; } + public long GetFileSize() + { + var info = new FileInfo(Filename); + return info.Length; + } + public void AssertIsEqual(TestFile? other) { if (other == null) Assert.Fail("TestFile is null."); if (other == this || other!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); + if (GetFileSize() != other.GetFileSize()) Assert.Fail("file sizes unequal?"); + using var stream1 = new FileStream(Filename, FileMode.Open, FileAccess.Read); using var stream2 = new FileStream(other.Filename, FileMode.Open, FileAccess.Read); diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 45c6b42..cc15a5d 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -2,40 +2,42 @@ { public interface IK8sManager { - IOnlineCodexNodes BringOnline(OfflineCodexNodes node); - IOfflineCodexNodes BringOffline(IOnlineCodexNodes node); + ICodexNodeGroup BringOnline(OfflineCodexNodes node); + IOfflineCodexNodes BringOffline(ICodexNodeGroup node); } public class K8sManager : IK8sManager { private readonly NumberSource onlineCodexNodeOrderNumberSource = new NumberSource(0); - private readonly List onlineCodexNodes = new List(); + private readonly List onlineCodexNodes = new List(); private readonly KnownK8sPods knownPods = new KnownK8sPods(); + private readonly TestLog log; private readonly IFileManager fileManager; - public K8sManager(IFileManager fileManager) + public K8sManager(TestLog log, IFileManager fileManager) { + this.log = log; this.fileManager = fileManager; } - public IOnlineCodexNodes BringOnline(OfflineCodexNodes offline) + public ICodexNodeGroup BringOnline(OfflineCodexNodes offline) { var online = CreateOnlineCodexNodes(offline); K8s().BringOnline(online, offline); - TestLog.Log($"{offline.NumberOfNodes} Codex nodes online."); + log.Log($"{online.Describe()} online."); return online; } - public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node) + public IOfflineCodexNodes BringOffline(ICodexNodeGroup node) { var online = GetAndRemoveActiveNodeFor(node); K8s().BringOffline(online); - TestLog.Log($"{online.Describe()} offline."); + log.Log($"{online.Describe()} offline."); return online.Origin; } @@ -50,11 +52,11 @@ K8s().FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler); } - private OnlineCodexNodes CreateOnlineCodexNodes(OfflineCodexNodes offline) + private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) { var containers = CreateContainers(offline.NumberOfNodes); - var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray(); - var result = new OnlineCodexNodes(onlineCodexNodeOrderNumberSource.GetNextNumber(), offline, this, online); + var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray(); + var result = new CodexNodeGroup(onlineCodexNodeOrderNumberSource.GetNextNumber(), offline, this, online); onlineCodexNodes.Add(result); return result; } @@ -67,9 +69,9 @@ return containers.ToArray(); } - private OnlineCodexNodes GetAndRemoveActiveNodeFor(IOnlineCodexNodes node) + private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node) { - var n = (OnlineCodexNodes)node; + var n = (CodexNodeGroup)node; onlineCodexNodes.Remove(n); return n; } diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 38a10d4..55b500d 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -21,7 +21,7 @@ namespace CodexDistTestCore client = new Kubernetes(config); } - public void BringOnline(OnlineCodexNodes online, OfflineCodexNodes offline) + public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline) { EnsureTestNamespace(); @@ -32,7 +32,7 @@ namespace CodexDistTestCore AssignActivePodNames(online); } - public void BringOffline(OnlineCodexNodes online) + public void BringOffline(CodexNodeGroup online) { var deploymentName = online.Deployment.Name(); DeleteDeployment(online); @@ -48,7 +48,7 @@ namespace CodexDistTestCore WaitUntilNamespaceDeleted(); } - public void FetchAllPodsLogs(OnlineCodexNodes[] onlines, IPodLogsHandler logHandler) + public void FetchAllPodsLogs(CodexNodeGroup[] onlines, IPodLogsHandler logHandler) { foreach (var online in onlines) { @@ -61,7 +61,7 @@ namespace CodexDistTestCore } } - private void AssignActivePodNames(OnlineCodexNodes online) + private void AssignActivePodNames(CodexNodeGroup online) { var pods = client.ListNamespacedPod(K8sNamespace); var podNames = pods.Items.Select(p => p.Name()); @@ -77,7 +77,7 @@ namespace CodexDistTestCore #region Waiting - private void WaitUntilOnline(OnlineCodexNodes online) + private void WaitUntilOnline(CodexNodeGroup online) { WaitUntil(() => { @@ -126,7 +126,7 @@ namespace CodexDistTestCore #region Service management - private void CreateService(OnlineCodexNodes online) + private void CreateService(CodexNodeGroup online) { var serviceSpec = new V1Service { @@ -143,7 +143,7 @@ namespace CodexDistTestCore online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); } - private List CreateServicePorts(OnlineCodexNodes online) + private List CreateServicePorts(CodexNodeGroup online) { var result = new List(); var containers = online.GetContainers(); @@ -160,7 +160,7 @@ namespace CodexDistTestCore return result; } - private void DeleteService(OnlineCodexNodes online) + private void DeleteService(CodexNodeGroup online) { if (online.Service == null) return; client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); @@ -171,7 +171,7 @@ namespace CodexDistTestCore #region Deployment management - private void CreateDeployment(OnlineCodexNodes online, OfflineCodexNodes offline) + private void CreateDeployment(CodexNodeGroup online, OfflineCodexNodes offline) { var deploymentSpec = new V1Deployment { @@ -201,7 +201,7 @@ namespace CodexDistTestCore online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); } - private List CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline) + private List CreateDeploymentContainers(CodexNodeGroup online, OfflineCodexNodes offline) { var result = new List(); var containers = online.GetContainers(); @@ -225,7 +225,7 @@ namespace CodexDistTestCore return result; } - private void DeleteDeployment(OnlineCodexNodes online) + private void DeleteDeployment(CodexNodeGroup online) { if (online.Deployment == null) return; client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace); diff --git a/CodexDistTestCore/OfflineCodexNodes.cs b/CodexDistTestCore/OfflineCodexNodes.cs index 10e0093..1b820e2 100644 --- a/CodexDistTestCore/OfflineCodexNodes.cs +++ b/CodexDistTestCore/OfflineCodexNodes.cs @@ -5,7 +5,7 @@ IOfflineCodexNodes WithLogLevel(CodexLogLevel level); IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node); IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota); - IOnlineCodexNodes BringOnline(); + ICodexNodeGroup BringOnline(); } public enum CodexLogLevel @@ -32,7 +32,7 @@ NumberOfNodes = numberOfNodes; } - public IOnlineCodexNodes BringOnline() + public ICodexNodeGroup BringOnline() { return k8SManager.BringOnline(this); } @@ -57,11 +57,15 @@ public string Describe() { - var result = ""; - if (LogLevel != null) result += $"LogLevel={LogLevel},"; - if (BootstrapNode != null) result += "BootstrapNode=set,"; - if (StorageQuota != null) result += $"StorageQuote={StorageQuota.SizeInBytes},"; - return result; + var args = string.Join(',', DescribeArgs()); + return $"{NumberOfNodes} CodexNodes with [{args}]"; + } + + private IEnumerable DescribeArgs() + { + if (LogLevel != null) yield return ($"LogLevel={LogLevel}"); + if (BootstrapNode != null) yield return ("BootstrapNode=set"); + if (StorageQuota != null) yield return ($"StorageQuote={StorageQuota.SizeInBytes}"); } } } diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index 54a1d88..1ce0c11 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -11,38 +11,46 @@ namespace CodexDistTestCore public class OnlineCodexNode : IOnlineCodexNode { + private readonly TestLog log; private readonly IFileManager fileManager; - public OnlineCodexNode(IFileManager fileManager, CodexNodeContainer environment) + public OnlineCodexNode(TestLog log, IFileManager fileManager, CodexNodeContainer container) { + this.log = log; this.fileManager = fileManager; - Container = environment; + Container = container; } public CodexNodeContainer Container { get; } public CodexDebugResponse GetDebugInfo() { - return Http().HttpGetJson("debug/info"); + var response = Http().HttpGetJson("debug/info"); + Log("Got DebugInfo with id: " + response.id); + return response; } public ContentId UploadFile(TestFile file) { + Log($"Uploading file of size {file.GetFileSize()}"); using var fileStream = File.OpenRead(file.Filename); var response = Http().HttpPostStream("upload", fileStream); if (response.StartsWith("Unable to store block")) { Assert.Fail("Node failed to store block."); } + Log($"Uploaded file. Received contentId: {response}"); return new ContentId(response); } public TestFile? DownloadContent(ContentId contentId) { + Log($"Downloading for contentId: {contentId}"); var file = fileManager.CreateEmptyTestFile(); using var fileStream = File.OpenWrite(file.Filename); using var downloadStream = Http().HttpGetStream("download/" + contentId.Id); downloadStream.CopyTo(fileStream); + Log($"Downloaded file of size {file.GetFileSize()}"); return file; } @@ -50,6 +58,11 @@ namespace CodexDistTestCore { return new Http(ip: "127.0.0.1", port: Container.ServicePort, baseUrl: "/api/codex/v1"); } + + private void Log(string msg) + { + log.Log($"Node {Container.Name}: {msg}"); + } } public class ContentId diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index 04cad1c..1ceb6b5 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -5,35 +5,28 @@ namespace CodexDistTestCore public class TestLog { public const string LogRoot = "D:/CodexTestLogs"; + private readonly LogFile file; - private static LogFile? file = null; - - // This is all way too static. It needs to be cleaned up. - public static void Log(string message) + public TestLog() { - file!.Write(message); - } - - public static void Error(string message) - { - Log($"[ERROR] {message}"); - } - - public static void BeginTest() - { - if (file != null) throw new InvalidOperationException("Test is already started!"); - var name = GetTestName(); file = new LogFile(name); Log($"Begin: {name}"); } - public static void EndTest(K8sManager k8sManager) + public void Log(string message) { - if (file == null) throw new InvalidOperationException("No test is started!"); + file.Write(message); + } + public void Error(string message) + { + Log($"[ERROR] {message}"); + } + public void EndTest(K8sManager k8sManager) + { var result = TestContext.CurrentContext.Result; Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); @@ -42,8 +35,6 @@ namespace CodexDistTestCore var logWriter = new PodLogWriter(file); logWriter.IncludeFullPodLogging(k8sManager); } - - file = null; } private static string GetTestName() @@ -65,14 +56,14 @@ namespace CodexDistTestCore public void IncludeFullPodLogging(K8sManager k8sManager) { - TestLog.Log("Full pod logging:"); + file.Write("Full pod logging:"); k8sManager.FetchAllPodsLogs(this); } public void Log(int id, string podDescription, Stream log) { var logFile = id.ToString().PadLeft(6, '0'); - TestLog.Log($"{podDescription} -->> {logFile}"); + file.Write($"{podDescription} -->> {logFile}"); LogRaw(podDescription, logFile); var reader = new StreamReader(log); var line = reader.ReadLine(); From f866e5f6484d59689cffe077be8c7d28a4f1289a Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 22 Mar 2023 09:50:24 +0100 Subject: [PATCH 05/12] Better logging still --- CodexDistTestCore/FileManager.cs | 3 --- CodexDistTestCore/OnlineCodexNode.cs | 15 ++++++++++----- CodexDistTestCore/TestLog.cs | 8 ++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/CodexDistTestCore/FileManager.cs b/CodexDistTestCore/FileManager.cs index 94abe7a..f884b0a 100644 --- a/CodexDistTestCore/FileManager.cs +++ b/CodexDistTestCore/FileManager.cs @@ -28,7 +28,6 @@ namespace CodexDistTestCore var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin")); File.Create(result.Filename).Close(); activeFiles.Add(result); - log.Log($"Created test file '{result.Filename}'."); return result; } @@ -86,8 +85,6 @@ namespace CodexDistTestCore if (other == null) Assert.Fail("TestFile is null."); if (other == this || other!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); - if (GetFileSize() != other.GetFileSize()) Assert.Fail("file sizes unequal?"); - using var stream1 = new FileStream(Filename, FileMode.Open, FileAccess.Read); using var stream2 = new FileStream(other.Filename, FileMode.Open, FileAccess.Read); diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index 1ce0c11..d8b1f6d 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -45,15 +45,20 @@ namespace CodexDistTestCore public TestFile? DownloadContent(ContentId contentId) { - Log($"Downloading for contentId: {contentId}"); + Log($"Downloading for contentId: {contentId.Id}"); var file = fileManager.CreateEmptyTestFile(); - using var fileStream = File.OpenWrite(file.Filename); - using var downloadStream = Http().HttpGetStream("download/" + contentId.Id); - downloadStream.CopyTo(fileStream); - Log($"Downloaded file of size {file.GetFileSize()}"); + DownloadToFile(contentId.Id, file); + Log($"Downloaded file of size {file.GetFileSize()} to {file.Filename}"); return file; } + private void DownloadToFile(string contentId, TestFile file) + { + using var fileStream = File.OpenWrite(file.Filename); + using var downloadStream = Http().HttpGetStream("download/" + contentId); + downloadStream.CopyTo(fileStream); + } + private Http Http() { return new Http(ip: "127.0.0.1", port: Container.ServicePort, baseUrl: "/api/codex/v1"); diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index 1ceb6b5..a2d33fd 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -30,8 +30,16 @@ namespace CodexDistTestCore var result = TestContext.CurrentContext.Result; Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); + + if (!string.IsNullOrEmpty(result.Message)) + { + Log(result.Message); + } + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) { + Log($"{result.StackTrace}"); + var logWriter = new PodLogWriter(file); logWriter.IncludeFullPodLogging(k8sManager); } From 79a513814fb797b3aebe4c05cb38eb91dd03ca0b Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 22 Mar 2023 10:38:10 +0100 Subject: [PATCH 06/12] Debugging multiple nodes per pod --- CodexDistTestCore/CodexNodeContainer.cs | 5 +- CodexDistTestCore/CodexNodeGroup.cs | 13 ++++- CodexDistTestCore/K8sOperations.cs | 3 +- CodexDistTestCore/TestLog.cs | 9 +++- codexnode-manifest.yml | 70 +++++++++++-------------- 5 files changed, 57 insertions(+), 43 deletions(-) diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs index 0926ab8..73f7847 100644 --- a/CodexDistTestCore/CodexNodeContainer.cs +++ b/CodexDistTestCore/CodexNodeContainer.cs @@ -2,10 +2,11 @@ { public class CodexNodeContainer { - public CodexNodeContainer(string name, int servicePort, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir) + public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir) { Name = name; ServicePort = servicePort; + ServicePortName = servicePortName; ApiPort = apiPort; ContainerPortName = containerPortName; DiscoveryPort = discoveryPort; @@ -15,6 +16,7 @@ public string Name { get; } public int ServicePort { get; } + public string ServicePortName { get; } public int ApiPort { get; } public string ContainerPortName { get; } public int DiscoveryPort { get; } @@ -34,6 +36,7 @@ return new CodexNodeContainer( name: $"codex-node{n}", servicePort: servicePortSource.GetNextNumber(), + servicePortName: $"node{n}", apiPort: codexPortSource.GetNextNumber(), containerPortName: $"api-{n}", discoveryPort: codexPortSource.GetNextNumber(), diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs index 23189e5..f929ace 100644 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -1,8 +1,9 @@ using k8s.Models; +using System.Collections; namespace CodexDistTestCore { - public interface ICodexNodeGroup + public interface ICodexNodeGroup : IEnumerable { IOfflineCodexNodes BringOffline(); IOnlineCodexNode this[int index] { get; } @@ -45,6 +46,16 @@ namespace CodexDistTestCore return Nodes.Select(n => n.Container).ToArray(); } + public IEnumerator GetEnumerator() + { + return Nodes.Cast().GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return Nodes.GetEnumerator(); + } + public V1ObjectMeta GetServiceMetadata() { return new V1ObjectMeta diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 55b500d..b88a184 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -151,8 +151,9 @@ namespace CodexDistTestCore { result.Add(new V1ServicePort { + Name = container.ServicePortName, Protocol = "TCP", - Port = 8080, + Port = container.ApiPort, TargetPort = container.ContainerPortName, NodePort = container.ServicePort }); diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index a2d33fd..95aad2b 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -49,7 +49,14 @@ namespace CodexDistTestCore { var test = TestContext.CurrentContext.Test; var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); - return $"{className}.{test.MethodName}"; + var args = FormatArguments(test); + return $"{className}.{test.MethodName}{args}"; + } + + private static string FormatArguments(TestContext.TestAdapter test) + { + if (test.Arguments == null || !test.Arguments.Any()) return ""; + return $"[{string.Join(',', test.Arguments)}]"; } } diff --git a/codexnode-manifest.yml b/codexnode-manifest.yml index c22d3ba..0b5bb94 100644 --- a/codexnode-manifest.yml +++ b/codexnode-manifest.yml @@ -1,28 +1,3 @@ -# apiVersion: v1 -# kind: Pod -# metadata: -# name: codex-pod -# spec: -# containers: -# - name: codex-node-container -# image: thatbenbierens/nim-codex:sha-c9a62de -# ports: -# - containerPort: 8080 - -# kind: PersistentVolumeClaim -# apiVersion: v1 -# metadata: -# name: codex-volume -# spec: -# accessModes: -# - ReadWriteOnce -# resources: -# requests: -# storage: 1Gi -# storageClassName: rbd - -# --- - apiVersion: v1 kind: Namespace metadata: @@ -48,21 +23,31 @@ spec: codex-node: dist-test spec: containers: - - name: codex-node - image: thatbenbierens/nim-codex:sha-c9a62de + - name: codex-node1 + image: thatbenbierens/nim-codex:sha-b204837 ports: - containerPort: 8080 - name: codex-api-port + name: codex-api-port1 env: - - name: LOG_LEVEL - value: WARN - # volumeMounts: - # - mountPath: /datadir - # name: codex-data - # volumes: - # - name: codex-data - # persistentVolumeClaim: - # claimName: codex-volume + - name: API_PORT + value: "8080" + - name: DATA_DIR + value: datadir1 + - name: codex-node2 + image: thatbenbierens/nim-codex:sha-b204837 + ports: + - containerPort: 8081 + name: codex-api-port2 + env: + - name: API_PORT + value: "8081" + - name: DATA_DIR + value: datadir2 + - name: DISC_PORT + value: "8091" + - name: LISTEN_ADDRS + value: "/ip4/0.0.0.0/tcp/8072" + --- apiVersion: v1 @@ -75,7 +60,14 @@ spec: selector: codex-node: dist-test ports: - - protocol: TCP + - name: "node1" + protocol: TCP port: 8080 - targetPort: codex-api-port + targetPort: codex-api-port1 nodePort: 30001 + - name: "node2" + protocol: TCP + port: 8081 + targetPort: codex-api-port2 + nodePort: 30002 + From 54f008a5e3a9c18d0606c3314ae9f268bde7ea24 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 22 Mar 2023 11:07:18 +0100 Subject: [PATCH 07/12] Adds test for 1000 codex nodes in one pod --- LongTests/BasicTests/SimpleTests.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/LongTests/BasicTests/SimpleTests.cs b/LongTests/BasicTests/SimpleTests.cs index 02dfaf1..44d1dca 100644 --- a/LongTests/BasicTests/SimpleTests.cs +++ b/LongTests/BasicTests/SimpleTests.cs @@ -22,5 +22,16 @@ namespace LongTests.BasicTests testFile.AssertIsEqual(downloadedFile); } + + [Test, UseLongTimeouts] + public void TestEnvironmentAddressSpaceTest() + { + var group = SetupCodexNodes(1000).BringOnline(); + + var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray(); + + Assert.That(nodeIds.Length, Is.EqualTo(nodeIds.Distinct().Count()), + "Not all created nodes provided a unique id."); + } } } From e9168dc1686eb4351ef4e9c5cf1807736bbecfe2 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 22 Mar 2023 11:27:51 +0100 Subject: [PATCH 08/12] Implements two-client-one-pod test --- CodexDistTestCore/Http.cs | 10 +++++-- CodexDistTestCore/OnlineCodexNode.cs | 41 +++++++++++++++++++++++----- Tests/BasicTests/SimpleTests.cs | 38 +++++++++++++------------- 3 files changed, 60 insertions(+), 29 deletions(-) diff --git a/CodexDistTestCore/Http.cs b/CodexDistTestCore/Http.cs index 1cff3c8..539877e 100644 --- a/CodexDistTestCore/Http.cs +++ b/CodexDistTestCore/Http.cs @@ -20,18 +20,22 @@ namespace CodexDistTestCore if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; } - public T HttpGetJson(string route) + public string HttpGetString(string route) { return Retry(() => { using var client = GetClient(); var url = GetUrl() + route; var result = Utils.Wait(client.GetAsync(url)); - var json = Utils.Wait(result.Content.ReadAsStringAsync()); - return JsonConvert.DeserializeObject(json)!; + return Utils.Wait(result.Content.ReadAsStringAsync()); }); } + public T HttpGetJson(string route) + { + return JsonConvert.DeserializeObject(HttpGetString(route))!; + } + public string HttpPostStream(string route, Stream stream) { return Retry(() => diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index d8b1f6d..55210e7 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -7,10 +7,14 @@ namespace CodexDistTestCore CodexDebugResponse GetDebugInfo(); ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); + void ConnectToPeer(IOnlineCodexNode node); } public class OnlineCodexNode : IOnlineCodexNode { + private const string SuccessfullyConnectedMessage = "Successfully connected to peer"; + private const string UploadFailedMessage = "Unable to store block"; + private readonly TestLog log; private readonly IFileManager fileManager; @@ -26,32 +30,55 @@ namespace CodexDistTestCore public CodexDebugResponse GetDebugInfo() { var response = Http().HttpGetJson("debug/info"); - Log("Got DebugInfo with id: " + response.id); + Log($"Got DebugInfo with id: {response.id}."); return response; } public ContentId UploadFile(TestFile file) { - Log($"Uploading file of size {file.GetFileSize()}"); + Log($"Uploading file of size {file.GetFileSize()}..."); using var fileStream = File.OpenRead(file.Filename); var response = Http().HttpPostStream("upload", fileStream); - if (response.StartsWith("Unable to store block")) + if (response.StartsWith(UploadFailedMessage)) { Assert.Fail("Node failed to store block."); } - Log($"Uploaded file. Received contentId: {response}"); + Log($"Uploaded file. Received contentId: {response}."); return new ContentId(response); } public TestFile? DownloadContent(ContentId contentId) { - Log($"Downloading for contentId: {contentId.Id}"); + Log($"Downloading for contentId: {contentId.Id}..."); var file = fileManager.CreateEmptyTestFile(); DownloadToFile(contentId.Id, file); - Log($"Downloaded file of size {file.GetFileSize()} to {file.Filename}"); + Log($"Downloaded file of size {file.GetFileSize()} to {file.Filename}."); return file; } + public void ConnectToPeer(IOnlineCodexNode node) + { + var peer = (OnlineCodexNode)node; + + Log($"Connecting to peer <{peer.Container.Name}>..."); + var peerInfo = node.GetDebugInfo(); + var peerId = peerInfo.id; + var peerMultiAddress = GetPeerMultiAddress(peer, peerInfo); + + var response = Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); + + Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes."); + Log($"Successfully connected to peer <{peer.Container.Name}>."); + } + + private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) + { + // Todo: If peer is in a different pod, we must replace 0.0.0.0 with the address of that pod! + + return peerInfo.addrs.First(); + // Todo: Is there a case where First address in list is not the way? + } + private void DownloadToFile(string contentId, TestFile file) { using var fileStream = File.OpenWrite(file.Filename); @@ -66,7 +93,7 @@ namespace CodexDistTestCore private void Log(string msg) { - log.Log($"Node {Container.Name}: {msg}"); + log.Log($"<{Container.Name}>: {msg}"); } } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 02fcecf..d780086 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -23,9 +23,9 @@ namespace Tests.BasicTests public void OneClientTest() { var primary = SetupCodexNodes(1) - .WithLogLevel(CodexLogLevel.Trace) - .WithStorageQuota(2.MB()) - .BringOnline()[0]; + .WithLogLevel(CodexLogLevel.Trace) + .WithStorageQuota(2.MB()) + .BringOnline()[0]; var testFile = GenerateTestFile(1.MB()); @@ -36,26 +36,26 @@ namespace Tests.BasicTests testFile.AssertIsEqual(downloadedFile); } - //[Test] - //public void TwoClientTest() - //{ - // var primary = SetupCodexNode() - // .WithLogLevel(CodexLogLevel.Trace) - // .WithStorageQuota(1024 * 1024 * 2) - // .BringOnline(); + [Test] + public void TwoClientOnePodTest() + { + var group = SetupCodexNodes(2) + .WithLogLevel(CodexLogLevel.Trace) + .WithStorageQuota(2.MB()) + .BringOnline(); - // var secondary = SetupCodexNode() - // .WithLogLevel(CodexLogLevel.Trace) - // .WithBootstrapNode(primary) - // .BringOnline(); + var primary = group[0]; + var secondary = group[1]; - // var testFile = GenerateTestFile(1024 * 1024); + primary.ConnectToPeer(secondary); - // var contentId = primary.UploadFile(testFile); + var testFile = GenerateTestFile(1.MB()); - // var downloadedFile = secondary.DownloadContent(contentId); + var contentId = primary.UploadFile(testFile); - // testFile.AssertIsEqual(downloadedFile); - //} + var downloadedFile = secondary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } } } From 5ccc3f0177437832e6ca649ed8e1beac14dfd76c Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 22 Mar 2023 13:52:01 +0100 Subject: [PATCH 09/12] Fixes pod logging for multiple codex nodes. --- CodexDistTestCore/CodexNodeGroup.cs | 2 +- CodexDistTestCore/K8sOperations.cs | 29 ++++++++++++++++------------ CodexDistTestCore/OnlineCodexNode.cs | 19 +++++++++++------- 3 files changed, 30 insertions(+), 20 deletions(-) diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs index f929ace..32c3cd2 100644 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -39,7 +39,7 @@ namespace CodexDistTestCore public OnlineCodexNode[] Nodes { get; } public V1Deployment? Deployment { get; set; } public V1Service? Service { get; set; } - public List ActivePodNames { get; } = new List(); + public string? PodName { get; set; } public CodexNodeContainer[] GetContainers() { diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index b88a184..adea879 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -50,29 +50,34 @@ namespace CodexDistTestCore public void FetchAllPodsLogs(CodexNodeGroup[] onlines, IPodLogsHandler logHandler) { + var logNumberSource = new NumberSource(0); foreach (var online in onlines) { - var nodeDescription = online.Describe(); - foreach (var podName in online.ActivePodNames) + foreach (var node in online) { - var stream = client.ReadNamespacedPodLog(podName, K8sNamespace); - logHandler.Log(online.OrderNumber, $"{nodeDescription}:{podName}", stream); + WritePodLogs(online, node, logHandler, logNumberSource); } } } + private void WritePodLogs(CodexNodeGroup online, IOnlineCodexNode node, IPodLogsHandler logHandler, NumberSource logNumberSource) + { + var n = (OnlineCodexNode)node; + var nodeDescription = $"{online.Describe()} contains {n.GetName()}"; + + var stream = client.ReadNamespacedPodLog(online.PodName, K8sNamespace, n.Container.Name); + logHandler.Log(logNumberSource.GetNextNumber(), nodeDescription, stream); + } + private void AssignActivePodNames(CodexNodeGroup online) { var pods = client.ListNamespacedPod(K8sNamespace); var podNames = pods.Items.Select(p => p.Name()); - foreach (var podName in podNames) - { - if (!knownPods.Contains(podName)) - { - knownPods.Add(podName); - online.ActivePodNames.Add(podName); - } - } + + var newPodNames = podNames.Where(p => !knownPods.Contains(p)).ToArray(); + Assert.That(newPodNames.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure."); + + online.PodName = newPodNames.Single(); } #region Waiting diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index 55210e7..baf304b 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -27,10 +27,15 @@ namespace CodexDistTestCore public CodexNodeContainer Container { get; } + public string GetName() + { + return $"<{Container.Name}>"; + } + public CodexDebugResponse GetDebugInfo() { var response = Http().HttpGetJson("debug/info"); - Log($"Got DebugInfo with id: {response.id}."); + Log($"Got DebugInfo with id: '{response.id}'."); return response; } @@ -43,16 +48,16 @@ namespace CodexDistTestCore { Assert.Fail("Node failed to store block."); } - Log($"Uploaded file. Received contentId: {response}."); + Log($"Uploaded file. Received contentId: '{response}'."); return new ContentId(response); } public TestFile? DownloadContent(ContentId contentId) { - Log($"Downloading for contentId: {contentId.Id}..."); + Log($"Downloading for contentId: '{contentId.Id}'..."); var file = fileManager.CreateEmptyTestFile(); DownloadToFile(contentId.Id, file); - Log($"Downloaded file of size {file.GetFileSize()} to {file.Filename}."); + Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'."); return file; } @@ -60,7 +65,7 @@ namespace CodexDistTestCore { var peer = (OnlineCodexNode)node; - Log($"Connecting to peer <{peer.Container.Name}>..."); + Log($"Connecting to peer {peer.GetName()}..."); var peerInfo = node.GetDebugInfo(); var peerId = peerInfo.id; var peerMultiAddress = GetPeerMultiAddress(peer, peerInfo); @@ -68,7 +73,7 @@ namespace CodexDistTestCore var response = Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes."); - Log($"Successfully connected to peer <{peer.Container.Name}>."); + Log($"Successfully connected to peer {peer.GetName()}."); } private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) @@ -93,7 +98,7 @@ namespace CodexDistTestCore private void Log(string msg) { - log.Log($"<{Container.Name}>: {msg}"); + log.Log($"{GetName()}: {msg}"); } } From 8fc573c4b5cd6605de515215d00f74d61812e1e5 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 22 Mar 2023 14:49:01 +0100 Subject: [PATCH 10/12] Setting up two-client-two-pod test --- CodexDistTestCore/CodexNodeContainer.cs | 33 ++++++++++++++++++++++--- CodexDistTestCore/CodexNodeGroup.cs | 16 +++++++++++- CodexDistTestCore/K8sManager.cs | 6 ++--- CodexDistTestCore/K8sOperations.cs | 21 ++++++++++------ CodexDistTestCore/OnlineCodexNode.cs | 14 ++++++++--- Tests/BasicTests/SimpleTests.cs | 19 ++++++++++++++ 6 files changed, 91 insertions(+), 18 deletions(-) diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs index 73f7847..fa0e348 100644 --- a/CodexDistTestCore/CodexNodeContainer.cs +++ b/CodexDistTestCore/CodexNodeContainer.cs @@ -24,19 +24,46 @@ public string DataDir { get; } } + public class CodexGroupNumberSource + { + private readonly NumberSource codexNodeGroupNumberSource = new NumberSource(0); + private readonly NumberSource groupContainerNameSource = new NumberSource(1); + private readonly NumberSource servicePortSource = new NumberSource(30001); + + public int GetNextCodexNodeGroupNumber() + { + return codexNodeGroupNumberSource.GetNextNumber(); + } + + public string GetNextServicePortName() + { + return $"node{groupContainerNameSource.GetNextNumber()}"; + } + + public int GetNextServicePort() + { + return servicePortSource.GetNextNumber(); + } + } + public class CodexNodeContainerFactory { private readonly NumberSource containerNameSource = new NumberSource(1); - private readonly NumberSource servicePortSource = new NumberSource(30001); private readonly NumberSource codexPortSource = new NumberSource(8080); + private readonly CodexGroupNumberSource groupContainerFactory; + + public CodexNodeContainerFactory(CodexGroupNumberSource groupContainerFactory) + { + this.groupContainerFactory = groupContainerFactory; + } public CodexNodeContainer CreateNext() { var n = containerNameSource.GetNextNumber(); return new CodexNodeContainer( name: $"codex-node{n}", - servicePort: servicePortSource.GetNextNumber(), - servicePortName: $"node{n}", + servicePort: groupContainerFactory.GetNextServicePort(), + servicePortName: groupContainerFactory.GetNextServicePortName(), apiPort: codexPortSource.GetNextNumber(), containerPortName: $"api-{n}", discoveryPort: codexPortSource.GetNextNumber(), diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs index 32c3cd2..8d8fda5 100644 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -19,6 +19,8 @@ namespace CodexDistTestCore Origin = origin; this.k8SManager = k8SManager; Nodes = nodes; + + foreach (var n in nodes) n.Group = this; } public IOnlineCodexNode this[int index] @@ -39,7 +41,7 @@ namespace CodexDistTestCore public OnlineCodexNode[] Nodes { get; } public V1Deployment? Deployment { get; set; } public V1Service? Service { get; set; } - public string? PodName { get; set; } + public PodInfo? PodInfo { get; set; } public CodexNodeContainer[] GetContainers() { @@ -84,4 +86,16 @@ namespace CodexDistTestCore return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}"; } } + + public class PodInfo + { + public PodInfo(string name, string ip) + { + Name = name; + Ip = ip; + } + + public string Name { get; } + public string Ip { get; } + } } diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index cc15a5d..b410b3f 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -8,7 +8,7 @@ public class K8sManager : IK8sManager { - private readonly NumberSource onlineCodexNodeOrderNumberSource = new NumberSource(0); + private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource(); private readonly List onlineCodexNodes = new List(); private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly TestLog log; @@ -56,14 +56,14 @@ { var containers = CreateContainers(offline.NumberOfNodes); var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray(); - var result = new CodexNodeGroup(onlineCodexNodeOrderNumberSource.GetNextNumber(), offline, this, online); + var result = new CodexNodeGroup(codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online); onlineCodexNodes.Add(result); return result; } private CodexNodeContainer[] CreateContainers(int number) { - var factory = new CodexNodeContainerFactory(); + var factory = new CodexNodeContainerFactory(codexGroupNumberSource); var containers = new List(); for (var i = 0; i < number; i++) containers.Add(factory.CreateNext()); return containers.ToArray(); diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index adea879..2d65fd6 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -29,7 +29,7 @@ namespace CodexDistTestCore CreateService(online); WaitUntilOnline(online); - AssignActivePodNames(online); + FetchPodInfo(online); } public void BringOffline(CodexNodeGroup online) @@ -65,19 +65,24 @@ namespace CodexDistTestCore var n = (OnlineCodexNode)node; var nodeDescription = $"{online.Describe()} contains {n.GetName()}"; - var stream = client.ReadNamespacedPodLog(online.PodName, K8sNamespace, n.Container.Name); + var stream = client.ReadNamespacedPodLog(online.PodInfo!.Name, K8sNamespace, n.Container.Name); logHandler.Log(logNumberSource.GetNextNumber(), nodeDescription, stream); } - private void AssignActivePodNames(CodexNodeGroup online) + private void FetchPodInfo(CodexNodeGroup online) { - var pods = client.ListNamespacedPod(K8sNamespace); - var podNames = pods.Items.Select(p => p.Name()); + var pods = client.ListNamespacedPod(K8sNamespace).Items; - var newPodNames = podNames.Where(p => !knownPods.Contains(p)).ToArray(); - Assert.That(newPodNames.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure."); + var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); + Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure."); - online.PodName = newPodNames.Single(); + var newPod = newPods.Single(); + online.PodInfo = new PodInfo(newPod.Name(), newPod.Status.PodIP); + + Assert.That(!string.IsNullOrEmpty(online.PodInfo.Name), "Invalid pod name received. Test infra failure."); + Assert.That(!string.IsNullOrEmpty(online.PodInfo.Ip), "Invalid pod IP received. Test infra failure."); + + knownPods.Add(newPod.Name()); } #region Waiting diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index baf304b..d783656 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -26,6 +26,7 @@ namespace CodexDistTestCore } public CodexNodeContainer Container { get; } + public CodexNodeGroup Group { get; internal set; } = null!; public string GetName() { @@ -78,10 +79,17 @@ namespace CodexDistTestCore private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) { - // Todo: If peer is in a different pod, we must replace 0.0.0.0 with the address of that pod! - - return peerInfo.addrs.First(); + var multiAddress = peerInfo.addrs.First(); // Todo: Is there a case where First address in list is not the way? + + if (Group == peer.Group) + { + return multiAddress; + } + + // The peer we want to connect is in a different pod. + // We must replace the default IP with the pod IP in the multiAddress. + return multiAddress.Replace("0.0.0.0", peer.Group.PodInfo!.Ip); } private void DownloadToFile(string contentId, TestFile file) diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index d780086..c052d64 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -47,6 +47,25 @@ namespace Tests.BasicTests var primary = group[0]; var secondary = group[1]; + PerformTwoClientTest(primary, secondary); + } + + [Test] + public void TwoClientTwoPodTest() + { + var primary = SetupCodexNodes(1) + .WithStorageQuota(2.MB()) + .BringOnline()[0]; + + var secondary = SetupCodexNodes(1) + .WithStorageQuota(2.MB()) + .BringOnline()[0]; + + PerformTwoClientTest(primary, secondary); + } + + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) + { primary.ConnectToPeer(secondary); var testFile = GenerateTestFile(1.MB()); From 1a31f324565b2fae1ea4f9781935322480d442c2 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 23 Mar 2023 12:35:03 +0100 Subject: [PATCH 11/12] Fixes timing issue in http retries --- CodexDistTestCore/Http.cs | 1 + CodexDistTestCore/K8sManager.cs | 14 ++--- CodexDistTestCore/K8sOperations.cs | 5 ++ CodexDistTestCore/Timing.cs | 10 ++-- .../{SimpleTests.cs => LargeFileTests.cs} | 17 ++---- LongTests/BasicTests/TestInfraTests.cs | 53 +++++++++++++++++++ 6 files changed, 75 insertions(+), 25 deletions(-) rename LongTests/BasicTests/{SimpleTests.cs => LargeFileTests.cs} (50%) create mode 100644 LongTests/BasicTests/TestInfraTests.cs diff --git a/CodexDistTestCore/Http.cs b/CodexDistTestCore/Http.cs index 539877e..fd7e31a 100644 --- a/CodexDistTestCore/Http.cs +++ b/CodexDistTestCore/Http.cs @@ -79,6 +79,7 @@ namespace CodexDistTestCore } catch (Exception exception) { + Timing.HttpCallRetryDelay(); retryCounter++; if (retryCounter > Timing.HttpCallRetryCount()) { diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index b410b3f..3530c96 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -24,7 +24,7 @@ { var online = CreateOnlineCodexNodes(offline); - K8s().BringOnline(online, offline); + K8s(k => k.BringOnline(online, offline)); log.Log($"{online.Describe()} online."); @@ -35,7 +35,7 @@ { var online = GetAndRemoveActiveNodeFor(node); - K8s().BringOffline(online); + K8s(k => k.BringOffline(online)); log.Log($"{online.Describe()} offline."); @@ -44,12 +44,12 @@ public void DeleteAllResources() { - K8s().DeleteAllResources(); + K8s(k => k.DeleteAllResources()); } public void FetchAllPodsLogs(IPodLogsHandler logHandler) { - K8s().FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler); + K8s(k => k.FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler)); } private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) @@ -76,9 +76,11 @@ return n; } - private K8sOperations K8s() + private void K8s(Action action) { - return new K8sOperations(knownPods); + var k8s = new K8sOperations(knownPods); + action(k8s); + k8s.Close(); } } } diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 2d65fd6..f9054b8 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -21,6 +21,11 @@ namespace CodexDistTestCore client = new Kubernetes(config); } + public void Close() + { + client.Dispose(); + } + public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline) { EnsureTestNamespace(); diff --git a/CodexDistTestCore/Timing.cs b/CodexDistTestCore/Timing.cs index 26c49b2..7ee0edf 100644 --- a/CodexDistTestCore/Timing.cs +++ b/CodexDistTestCore/Timing.cs @@ -25,9 +25,9 @@ namespace CodexDistTestCore return GetTimes().HttpCallRetryCount(); } - public static void RetryDelay() + public static void HttpCallRetryDelay() { - Utils.Sleep(GetTimes().RetryDelay()); + Utils.Sleep(GetTimes().HttpCallRetryDelay()); } public static void WaitForK8sServiceDelay() @@ -52,7 +52,7 @@ namespace CodexDistTestCore { TimeSpan HttpCallTimeout(); int HttpCallRetryCount(); - TimeSpan RetryDelay(); + TimeSpan HttpCallRetryDelay(); TimeSpan WaitForK8sServiceDelay(); TimeSpan K8sOperationTimeout(); } @@ -69,7 +69,7 @@ namespace CodexDistTestCore return 5; } - public TimeSpan RetryDelay() + public TimeSpan HttpCallRetryDelay() { return TimeSpan.FromSeconds(3); } @@ -97,7 +97,7 @@ namespace CodexDistTestCore return 2; } - public TimeSpan RetryDelay() + public TimeSpan HttpCallRetryDelay() { return TimeSpan.FromMinutes(5); } diff --git a/LongTests/BasicTests/SimpleTests.cs b/LongTests/BasicTests/LargeFileTests.cs similarity index 50% rename from LongTests/BasicTests/SimpleTests.cs rename to LongTests/BasicTests/LargeFileTests.cs index 44d1dca..b8ab20e 100644 --- a/LongTests/BasicTests/SimpleTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -4,17 +4,17 @@ using NUnit.Framework; namespace LongTests.BasicTests { [TestFixture] - public class SimpleTests : DistTest + public class LargeFileTests : DistTest { [Test, UseLongTimeouts] public void OneClientLargeFileTest() { var primary = SetupCodexNodes(1) .WithLogLevel(CodexLogLevel.Warn) - .WithStorageQuota(10.GB()) + .WithStorageQuota(20.GB()) .BringOnline()[0]; - var testFile = GenerateTestFile(1.GB()); + var testFile = GenerateTestFile(10.GB()); var contentId = primary.UploadFile(testFile); @@ -22,16 +22,5 @@ namespace LongTests.BasicTests testFile.AssertIsEqual(downloadedFile); } - - [Test, UseLongTimeouts] - public void TestEnvironmentAddressSpaceTest() - { - var group = SetupCodexNodes(1000).BringOnline(); - - var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray(); - - Assert.That(nodeIds.Length, Is.EqualTo(nodeIds.Distinct().Count()), - "Not all created nodes provided a unique id."); - } } } diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs new file mode 100644 index 0000000..1e2f62e --- /dev/null +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -0,0 +1,53 @@ +using CodexDistTestCore; +using NUnit.Framework; + +namespace LongTests.BasicTests +{ + public class TestInfraTests : DistTest + { + [Test, UseLongTimeouts] + public void TestInfraShouldHave1000AddressSpacesPerPod() + { + var group = SetupCodexNodes(1000).BringOnline(); + + var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray(); + + Assert.That(nodeIds.Length, Is.EqualTo(nodeIds.Distinct().Count()), + "Not all created nodes provided a unique id."); + } + + [Test, UseLongTimeouts] + public void TestInfraSupportsManyConcurrentPods() + { + for (var i = 0; i < 20; i++) + { + var n = SetupCodexNodes(1).BringOnline()[0]; + + Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().id)); + } + } + + [Test, UseLongTimeouts] + public void DownloadConsistencyTest() + { + var primary = SetupCodexNodes(1) + .WithLogLevel(CodexLogLevel.Trace) + .WithStorageQuota(2.MB()) + .BringOnline()[0]; + + var testFile = GenerateTestFile(1.MB()); + + var contentId = primary.UploadFile(testFile); + + var files = new List(); + for (var i = 0; i < 100; i++) + { + files.Add(primary.DownloadContent(contentId)); + } + + Assert.That(files.All(f => f != null)); + Assert.That(files.All(f => f!.GetFileSize() == testFile.GetFileSize())); + foreach (var file in files) file!.AssertIsEqual(testFile); + } + } +} From 6e068882ca1e6c5398cc9e50a820068a79cfc498 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 23 Mar 2023 12:36:36 +0100 Subject: [PATCH 12/12] example yml --- codexnode-manifest.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/codexnode-manifest.yml b/codexnode-manifest.yml index 0b5bb94..dafddec 100644 --- a/codexnode-manifest.yml +++ b/codexnode-manifest.yml @@ -27,26 +27,30 @@ spec: image: thatbenbierens/nim-codex:sha-b204837 ports: - containerPort: 8080 - name: codex-api-port1 + name: api-1 env: - name: API_PORT value: "8080" - name: DATA_DIR value: datadir1 + - name: DISC_PORT + value: "8081" + - name: LISTEN_ADDRS + value: "/ip4/0.0.0.0/tcp/8082" - name: codex-node2 image: thatbenbierens/nim-codex:sha-b204837 ports: - - containerPort: 8081 - name: codex-api-port2 + - containerPort: 8083 + name: api-2 env: - name: API_PORT - value: "8081" + value: "8083" - name: DATA_DIR value: datadir2 - name: DISC_PORT - value: "8091" + value: "8084" - name: LISTEN_ADDRS - value: "/ip4/0.0.0.0/tcp/8072" + value: "/ip4/0.0.0.0/tcp/8085" --- @@ -63,11 +67,11 @@ spec: - name: "node1" protocol: TCP port: 8080 - targetPort: codex-api-port1 + targetPort: api-1 nodePort: 30001 - name: "node2" protocol: TCP - port: 8081 - targetPort: codex-api-port2 + port: 8083 + targetPort: api-2 nodePort: 30002