From f3a5ed397643ec75b02d198471045e78d1940b64 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 21 Mar 2023 15:44:21 +0100 Subject: [PATCH] cleanup --- CodexDistTestCore/ActiveDeployment.cs | 49 ----------- CodexDistTestCore/K8sManager.cs | 121 +++++++++++++------------- CodexDistTestCore/OnlineCodexNode.cs | 7 +- CodexDistTestCore/OnlineCodexNodes.cs | 53 +++++++++-- CodexDistTestCore/TestLog.cs | 9 +- 5 files changed, 116 insertions(+), 123 deletions(-) delete mode 100644 CodexDistTestCore/ActiveDeployment.cs diff --git a/CodexDistTestCore/ActiveDeployment.cs b/CodexDistTestCore/ActiveDeployment.cs deleted file mode 100644 index f35642e..0000000 --- a/CodexDistTestCore/ActiveDeployment.cs +++ /dev/null @@ -1,49 +0,0 @@ -using k8s.Models; - -namespace CodexDistTestCore -{ - public class ActiveDeployment - { - public ActiveDeployment(OfflineCodexNodes origin, int orderNumber, CodexNodeContainer[] containers) - { - Origin = origin; - Containers = containers; - SelectorName = orderNumber.ToString().PadLeft(6, '0'); - } - - public OfflineCodexNodes Origin { get; } - public CodexNodeContainer[] Containers { get; } - public string SelectorName { get; } - public V1Deployment? Deployment { get; set; } - public V1Service? Service { get; set; } - public List ActivePodNames { get; } = new List(); - - public V1ObjectMeta GetServiceMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-entrypoint-" + SelectorName, - NamespaceProperty = K8sManager.K8sNamespace - }; - } - - public V1ObjectMeta GetDeploymentMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-node-" + SelectorName, - NamespaceProperty = K8sManager.K8sNamespace - }; - } - - public Dictionary GetSelector() - { - return new Dictionary { { "codex-test-node", "dist-test-" + SelectorName } }; - } - - public string Describe() - { - return $"CodexNode{SelectorName}-{Origin.Describe()}"; - } - } -} diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 004782a..726d225 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -15,7 +15,7 @@ namespace CodexDistTestCore public const string K8sNamespace = "codex-test-namespace"; private readonly CodexDockerImage dockerImage = new CodexDockerImage(); private readonly NumberSource activeDeploymentOrderNumberSource = new NumberSource(0); - private readonly Dictionary activeDeployments = new Dictionary(); + private readonly List activeCodexNodes = new List(); private readonly List knownActivePodNames = new List(); private readonly IFileManager fileManager; @@ -24,37 +24,33 @@ namespace CodexDistTestCore this.fileManager = fileManager; } - public IOnlineCodexNodes BringOnline(OfflineCodexNodes node) + public IOnlineCodexNodes BringOnline(OfflineCodexNodes offline) { var client = CreateClient(); EnsureTestNamespace(client); - var factory = new CodexNodeContainerFactory(); - var list = new List(); - var containers = new List(); - for (var i = 0; i < node.NumberOfNodes; i++) - { - var container = factory.CreateNext(); - containers.Add(container); + var containers = CreateContainers(offline.NumberOfNodes); + var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray(); + var result = new OnlineCodexNodes(activeDeploymentOrderNumberSource.GetNextNumber(), offline, this, online); + activeCodexNodes.Add(result); - var codexNode = new OnlineCodexNode(fileManager, container); - list.Add(codexNode); - } + CreateDeployment(client, result, offline); + CreateService(result, client); - var activeDeployment = new ActiveDeployment(node, activeDeploymentOrderNumberSource.GetNextNumber(), containers.ToArray()); - - var result = new OnlineCodexNodes(this, list.ToArray()); - activeDeployments.Add(result, activeDeployment); - - CreateDeployment(activeDeployment, client, node); - CreateService(activeDeployment, client); - - WaitUntilOnline(activeDeployment, client); - TestLog.Log($"{node.NumberOfNodes} Codex nodes online."); + WaitUntilOnline(result, client); + TestLog.Log($"{offline.NumberOfNodes} Codex nodes online."); return result; } + private CodexNodeContainer[] CreateContainers(int number) + { + var factory = new CodexNodeContainerFactory(); + var containers = new List(); + for (var i = 0; i < number; i++) containers.Add(factory.CreateNext()); + return containers.ToArray(); + } + public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node) { var client = CreateClient(); @@ -79,40 +75,40 @@ namespace CodexDistTestCore WaitUntilNamespaceDeleted(client); } - public void FetchAllPodsLogs(Action onLog) + public void FetchAllPodsLogs(Action onLog) { var client = CreateClient(); - foreach (var node in activeDeployments.Values) + foreach (var node in activeCodexNodes) { var nodeDescription = node.Describe(); foreach (var podName in node.ActivePodNames) { var stream = client.ReadNamespacedPodLog(podName, K8sNamespace); - onLog(node.SelectorName, $"{nodeDescription}:{podName}", stream); + onLog(node.OrderNumber, $"{nodeDescription}:{podName}", stream); } } } - private void BringOffline(ActiveDeployment activeNode, Kubernetes client) + private void BringOffline(OnlineCodexNodes online, Kubernetes client) { - DeleteDeployment(activeNode, client); - DeleteService(activeNode, client); + DeleteDeployment(client, online); + DeleteService(client, online); } #region Waiting - private void WaitUntilOnline(ActiveDeployment activeNode, Kubernetes client) + private void WaitUntilOnline(OnlineCodexNodes online, Kubernetes client) { WaitUntil(() => { - activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), K8sNamespace); - return activeNode.Deployment?.Status.AvailableReplicas != null && activeNode.Deployment.Status.AvailableReplicas > 0; + online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); + return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; }); - AssignActivePodNames(activeNode, client); + AssignActivePodNames(online, client); } - private void AssignActivePodNames(ActiveDeployment activeNode, Kubernetes client) + private void AssignActivePodNames(OnlineCodexNodes online, Kubernetes client) { var pods = client.ListNamespacedPod(K8sNamespace); var podNames = pods.Items.Select(p => p.Name()); @@ -121,7 +117,7 @@ namespace CodexDistTestCore if (!knownActivePodNames.Contains(podName)) { knownActivePodNames.Add(podName); - activeNode.ActivePodNames.Add(podName); + online.ActivePodNames.Add(podName); } } } @@ -166,27 +162,28 @@ namespace CodexDistTestCore #region Service management - private void CreateService(ActiveDeployment activeDeployment, Kubernetes client) + private void CreateService(OnlineCodexNodes online, Kubernetes client) { var serviceSpec = new V1Service { ApiVersion = "v1", - Metadata = activeDeployment.GetServiceMetadata(), + Metadata = online.GetServiceMetadata(), Spec = new V1ServiceSpec { Type = "NodePort", - Selector = activeDeployment.GetSelector(), - Ports = CreateServicePorts(activeDeployment) + Selector = online.GetSelector(), + Ports = CreateServicePorts(online) } }; - activeDeployment.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); + online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); } - private List CreateServicePorts(ActiveDeployment activeDeployment) + private List CreateServicePorts(OnlineCodexNodes online) { var result = new List(); - foreach (var container in activeDeployment.Containers) + var containers = online.GetContainers(); + foreach (var container in containers) { result.Add(new V1ServicePort { @@ -199,51 +196,52 @@ namespace CodexDistTestCore return result; } - private void DeleteService(ActiveDeployment node, Kubernetes client) + private void DeleteService(Kubernetes client, OnlineCodexNodes online) { - if (node.Service == null) return; - client.DeleteNamespacedService(node.Service.Name(), K8sNamespace); - node.Service = null; + if (online.Service == null) return; + client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); + online.Service = null; } #endregion #region Deployment management - private void CreateDeployment(ActiveDeployment node, Kubernetes client, OfflineCodexNodes codexNode) + private void CreateDeployment(Kubernetes client, OnlineCodexNodes online, OfflineCodexNodes offline) { var deploymentSpec = new V1Deployment { ApiVersion = "apps/v1", - Metadata = node.GetDeploymentMetadata(), + Metadata = online.GetDeploymentMetadata(), Spec = new V1DeploymentSpec { Replicas = 1, Selector = new V1LabelSelector { - MatchLabels = node.GetSelector() + MatchLabels = online.GetSelector() }, Template = new V1PodTemplateSpec { Metadata = new V1ObjectMeta { - Labels = node.GetSelector() + Labels = online.GetSelector() }, Spec = new V1PodSpec { - Containers = CreateDeploymentContainers(node, codexNode) + Containers = CreateDeploymentContainers(online, offline) } } } }; - node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); + online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); } - private List CreateDeploymentContainers(ActiveDeployment node,OfflineCodexNodes codexNode) + private List CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline) { var result = new List(); - foreach (var container in node.Containers) + var containers = online.GetContainers(); + foreach (var container in containers) { result.Add(new V1Container { @@ -257,17 +255,17 @@ namespace CodexDistTestCore Name = container.ContainerPortName } }, - Env = dockerImage.CreateEnvironmentVariables(codexNode, container) + Env = dockerImage.CreateEnvironmentVariables(offline, container) }); } return result; } - private void DeleteDeployment(ActiveDeployment node, Kubernetes client) + private void DeleteDeployment(Kubernetes client, OnlineCodexNodes online) { - if (node.Deployment == null) return; - client.DeleteNamespacedDeployment(node.Deployment.Name(), K8sNamespace); - node.Deployment = null; + if (online.Deployment == null) return; + client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace); + online.Deployment = null; } #endregion @@ -312,12 +310,11 @@ namespace CodexDistTestCore return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); } - private ActiveDeployment GetAndRemoveActiveNodeFor(IOnlineCodexNodes node) + private OnlineCodexNodes GetAndRemoveActiveNodeFor(IOnlineCodexNodes node) { var n = (OnlineCodexNodes)node; - var activeNode = activeDeployments[n]; - activeDeployments.Remove(n); - return activeNode; + activeCodexNodes.Remove(n); + return n; } } } diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index 8acef06..54a1d88 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -12,14 +12,15 @@ namespace CodexDistTestCore public class OnlineCodexNode : IOnlineCodexNode { private readonly IFileManager fileManager; - private readonly CodexNodeContainer environment; public OnlineCodexNode(IFileManager fileManager, CodexNodeContainer environment) { this.fileManager = fileManager; - this.environment = environment; + Container = environment; } + public CodexNodeContainer Container { get; } + public CodexDebugResponse GetDebugInfo() { return Http().HttpGetJson("debug/info"); @@ -47,7 +48,7 @@ namespace CodexDistTestCore private Http Http() { - return new Http(ip: "127.0.0.1", port: environment.ServicePort, baseUrl: "/api/codex/v1"); + return new Http(ip: "127.0.0.1", port: Container.ServicePort, baseUrl: "/api/codex/v1"); } } diff --git a/CodexDistTestCore/OnlineCodexNodes.cs b/CodexDistTestCore/OnlineCodexNodes.cs index 0f0280a..119cf1f 100644 --- a/CodexDistTestCore/OnlineCodexNodes.cs +++ b/CodexDistTestCore/OnlineCodexNodes.cs @@ -1,4 +1,6 @@ -namespace CodexDistTestCore +using k8s.Models; + +namespace CodexDistTestCore { public interface IOnlineCodexNodes { @@ -9,25 +11,66 @@ public class OnlineCodexNodes : IOnlineCodexNodes { private readonly IK8sManager k8SManager; - private readonly IOnlineCodexNode[] nodes; - public OnlineCodexNodes(IK8sManager k8SManager, IOnlineCodexNode[] nodes) + public OnlineCodexNodes(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes) { + OrderNumber = orderNumber; + Origin = origin; this.k8SManager = k8SManager; - this.nodes = nodes; + Nodes = nodes; } public IOnlineCodexNode this[int index] { get { - return nodes[index]; + return Nodes[index]; } } + public int OrderNumber { get; } + public OfflineCodexNodes Origin { get; } + public OnlineCodexNode[] Nodes { get; } + public V1Deployment? Deployment { get; set; } + public V1Service? Service { get; set; } + public List ActivePodNames { get; } = new List(); + public IOfflineCodexNodes BringOffline() { return k8SManager.BringOffline(this); } + + public CodexNodeContainer[] GetContainers() + { + return Nodes.Select(n => n.Container).ToArray(); + } + + public V1ObjectMeta GetServiceMetadata() + { + return new V1ObjectMeta + { + Name = "codex-test-entrypoint-" + OrderNumber, + NamespaceProperty = K8sManager.K8sNamespace + }; + } + + public V1ObjectMeta GetDeploymentMetadata() + { + return new V1ObjectMeta + { + Name = "codex-test-node-" + OrderNumber, + NamespaceProperty = K8sManager.K8sNamespace + }; + } + + public Dictionary GetSelector() + { + return new Dictionary { { "codex-test-node", "dist-test-" + OrderNumber } }; + } + + public string Describe() + { + return $"CodexNode{OrderNumber}-{Origin.Describe()}"; + } } } diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index 3854cd1..e376204 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -62,15 +62,16 @@ namespace CodexDistTestCore k8sManager.FetchAllPodsLogs(WritePodLog); } - private static void WritePodLog(string id, string nodeDescription, Stream stream) + private static void WritePodLog(int id, string nodeDescription, Stream stream) { - Log($"{nodeDescription} -->> {id}"); - LogRaw(nodeDescription, id); + var logFile = id.ToString().PadLeft(6, '0'); + Log($"{nodeDescription} -->> {logFile}"); + LogRaw(nodeDescription, logFile); var reader = new StreamReader(stream); var line = reader.ReadLine(); while (line != null) { - LogRaw(line, id); + LogRaw(line, logFile); line = reader.ReadLine(); } }