diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 726d225..45c6b42 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -1,8 +1,4 @@ -using k8s; -using k8s.Models; -using NUnit.Framework; - -namespace CodexDistTestCore +namespace CodexDistTestCore { public interface IK8sManager { @@ -12,11 +8,9 @@ namespace CodexDistTestCore public class K8sManager : IK8sManager { - public const string K8sNamespace = "codex-test-namespace"; - private readonly CodexDockerImage dockerImage = new CodexDockerImage(); - private readonly NumberSource activeDeploymentOrderNumberSource = new NumberSource(0); - private readonly List activeCodexNodes = new List(); - private readonly List knownActivePodNames = new List(); + private readonly NumberSource onlineCodexNodeOrderNumberSource = new NumberSource(0); + private readonly List onlineCodexNodes = new List(); + private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly IFileManager fileManager; public K8sManager(IFileManager fileManager) @@ -26,20 +20,42 @@ namespace CodexDistTestCore public IOnlineCodexNodes BringOnline(OfflineCodexNodes offline) { - var client = CreateClient(); - EnsureTestNamespace(client); + var online = CreateOnlineCodexNodes(offline); - var containers = CreateContainers(offline.NumberOfNodes); - var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray(); - var result = new OnlineCodexNodes(activeDeploymentOrderNumberSource.GetNextNumber(), offline, this, online); - activeCodexNodes.Add(result); + K8s().BringOnline(online, offline); - CreateDeployment(client, result, offline); - CreateService(result, client); - - WaitUntilOnline(result, client); TestLog.Log($"{offline.NumberOfNodes} Codex nodes online."); + return online; + } + + public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node) + { + var online = GetAndRemoveActiveNodeFor(node); + + K8s().BringOffline(online); + + TestLog.Log($"{online.Describe()} offline."); + + return online.Origin; + } + + public void DeleteAllResources() + { + K8s().DeleteAllResources(); + } + + public void FetchAllPodsLogs(IPodLogsHandler logHandler) + { + K8s().FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler); + } + + private OnlineCodexNodes CreateOnlineCodexNodes(OfflineCodexNodes offline) + { + var containers = CreateContainers(offline.NumberOfNodes); + var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray(); + var result = new OnlineCodexNodes(onlineCodexNodeOrderNumberSource.GetNextNumber(), offline, this, online); + onlineCodexNodes.Add(result); return result; } @@ -51,270 +67,16 @@ namespace CodexDistTestCore return containers.ToArray(); } - public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node) - { - var client = CreateClient(); - - var activeNode = GetAndRemoveActiveNodeFor(node); - - var deploymentName = activeNode.Deployment.Name(); - BringOffline(activeNode, client); - WaitUntilOffline(deploymentName, client); - TestLog.Log($"{activeNode.Describe()} offline."); - - return activeNode.Origin; - } - - public void DeleteAllResources() - { - var client = CreateClient(); - - DeleteNamespace(client); - - WaitUntilZeroPods(client); - WaitUntilNamespaceDeleted(client); - } - - public void FetchAllPodsLogs(Action onLog) - { - var client = CreateClient(); - foreach (var node in activeCodexNodes) - { - var nodeDescription = node.Describe(); - foreach (var podName in node.ActivePodNames) - { - var stream = client.ReadNamespacedPodLog(podName, K8sNamespace); - onLog(node.OrderNumber, $"{nodeDescription}:{podName}", stream); - } - } - } - - private void BringOffline(OnlineCodexNodes online, Kubernetes client) - { - DeleteDeployment(client, online); - DeleteService(client, online); - } - - #region Waiting - - private void WaitUntilOnline(OnlineCodexNodes online, Kubernetes client) - { - WaitUntil(() => - { - online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); - return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; - }); - - AssignActivePodNames(online, client); - } - - private void AssignActivePodNames(OnlineCodexNodes online, Kubernetes client) - { - var pods = client.ListNamespacedPod(K8sNamespace); - var podNames = pods.Items.Select(p => p.Name()); - foreach (var podName in podNames) - { - if (!knownActivePodNames.Contains(podName)) - { - knownActivePodNames.Add(podName); - online.ActivePodNames.Add(podName); - } - } - } - - private void WaitUntilOffline(string deploymentName, Kubernetes client) - { - WaitUntil(() => - { - var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); - return deployment == null || deployment.Status.AvailableReplicas == 0; - }); - } - - private void WaitUntilZeroPods(Kubernetes client) - { - WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); - } - - private void WaitUntilNamespaceDeleted(Kubernetes client) - { - WaitUntil(() => !IsTestNamespaceOnline(client)); - } - - private void WaitUntil(Func predicate) - { - var start = DateTime.UtcNow; - var state = predicate(); - while (!state) - { - if (DateTime.UtcNow - start > Timing.K8sOperationTimeout()) - { - Assert.Fail("K8s operation timed out."); - throw new TimeoutException(); - } - - Timing.WaitForK8sServiceDelay(); - state = predicate(); - } - } - - #endregion - - #region Service management - - private void CreateService(OnlineCodexNodes online, Kubernetes client) - { - var serviceSpec = new V1Service - { - ApiVersion = "v1", - Metadata = online.GetServiceMetadata(), - Spec = new V1ServiceSpec - { - Type = "NodePort", - Selector = online.GetSelector(), - Ports = CreateServicePorts(online) - } - }; - - online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); - } - - private List CreateServicePorts(OnlineCodexNodes online) - { - var result = new List(); - var containers = online.GetContainers(); - foreach (var container in containers) - { - result.Add(new V1ServicePort - { - Protocol = "TCP", - Port = 8080, - TargetPort = container.ContainerPortName, - NodePort = container.ServicePort - }); - } - return result; - } - - private void DeleteService(Kubernetes client, OnlineCodexNodes online) - { - if (online.Service == null) return; - client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); - online.Service = null; - } - - #endregion - - #region Deployment management - - private void CreateDeployment(Kubernetes client, OnlineCodexNodes online, OfflineCodexNodes offline) - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = online.GetDeploymentMetadata(), - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = online.GetSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = online.GetSelector() - }, - Spec = new V1PodSpec - { - Containers = CreateDeploymentContainers(online, offline) - } - } - } - }; - - online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); - } - - private List CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline) - { - var result = new List(); - var containers = online.GetContainers(); - foreach (var container in containers) - { - result.Add(new V1Container - { - Name = container.Name, - Image = dockerImage.GetImageTag(), - Ports = new List - { - new V1ContainerPort - { - ContainerPort = container.ApiPort, - Name = container.ContainerPortName - } - }, - Env = dockerImage.CreateEnvironmentVariables(offline, container) - }); - } - return result; - } - - private void DeleteDeployment(Kubernetes client, OnlineCodexNodes online) - { - if (online.Deployment == null) return; - client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace); - online.Deployment = null; - } - - #endregion - - #region Namespace management - - private void EnsureTestNamespace(Kubernetes client) - { - if (IsTestNamespaceOnline(client)) return; - - var namespaceSpec = new V1Namespace - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = K8sNamespace, - Labels = new Dictionary { { "name", K8sNamespace } } - } - }; - client.CreateNamespace(namespaceSpec); - } - - private void DeleteNamespace(Kubernetes client) - { - if (IsTestNamespaceOnline(client)) - { - client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0); - } - } - - #endregion - - private static Kubernetes CreateClient() - { - // todo: If the default KubeConfig file does not suffice, change it here: - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(); - return new Kubernetes(config); - } - - private static bool IsTestNamespaceOnline(Kubernetes client) - { - return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); - } - private OnlineCodexNodes GetAndRemoveActiveNodeFor(IOnlineCodexNodes node) { var n = (OnlineCodexNodes)node; - activeCodexNodes.Remove(n); + onlineCodexNodes.Remove(n); return n; } + + private K8sOperations K8s() + { + return new K8sOperations(knownPods); + } } } diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs new file mode 100644 index 0000000..38a10d4 --- /dev/null +++ b/CodexDistTestCore/K8sOperations.cs @@ -0,0 +1,270 @@ +using k8s; +using k8s.Models; +using NUnit.Framework; + +namespace CodexDistTestCore +{ + public class K8sOperations + { + public const string K8sNamespace = "codex-test-namespace"; + + private readonly CodexDockerImage dockerImage = new CodexDockerImage(); + private readonly Kubernetes client; + private readonly KnownK8sPods knownPods; + + public K8sOperations(KnownK8sPods knownPods) + { + this.knownPods = knownPods; + + // todo: If the default KubeConfig file does not suffice, change it here: + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(); + client = new Kubernetes(config); + } + + public void BringOnline(OnlineCodexNodes online, OfflineCodexNodes offline) + { + EnsureTestNamespace(); + + CreateDeployment(online, offline); + CreateService(online); + + WaitUntilOnline(online); + AssignActivePodNames(online); + } + + public void BringOffline(OnlineCodexNodes online) + { + var deploymentName = online.Deployment.Name(); + DeleteDeployment(online); + DeleteService(online); + WaitUntilOffline(deploymentName); + } + + public void DeleteAllResources() + { + DeleteNamespace(); + + WaitUntilZeroPods(); + WaitUntilNamespaceDeleted(); + } + + public void FetchAllPodsLogs(OnlineCodexNodes[] onlines, IPodLogsHandler logHandler) + { + foreach (var online in onlines) + { + var nodeDescription = online.Describe(); + foreach (var podName in online.ActivePodNames) + { + var stream = client.ReadNamespacedPodLog(podName, K8sNamespace); + logHandler.Log(online.OrderNumber, $"{nodeDescription}:{podName}", stream); + } + } + } + + private void AssignActivePodNames(OnlineCodexNodes online) + { + var pods = client.ListNamespacedPod(K8sNamespace); + var podNames = pods.Items.Select(p => p.Name()); + foreach (var podName in podNames) + { + if (!knownPods.Contains(podName)) + { + knownPods.Add(podName); + online.ActivePodNames.Add(podName); + } + } + } + + #region Waiting + + private void WaitUntilOnline(OnlineCodexNodes online) + { + WaitUntil(() => + { + online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); + return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; + }); + } + + private void WaitUntilOffline(string deploymentName) + { + WaitUntil(() => + { + var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); + return deployment == null || deployment.Status.AvailableReplicas == 0; + }); + } + + private void WaitUntilZeroPods() + { + WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); + } + + private void WaitUntilNamespaceDeleted() + { + WaitUntil(() => !IsTestNamespaceOnline()); + } + + private void WaitUntil(Func predicate) + { + var start = DateTime.UtcNow; + var state = predicate(); + while (!state) + { + if (DateTime.UtcNow - start > Timing.K8sOperationTimeout()) + { + Assert.Fail("K8s operation timed out."); + throw new TimeoutException(); + } + + Timing.WaitForK8sServiceDelay(); + state = predicate(); + } + } + + #endregion + + #region Service management + + private void CreateService(OnlineCodexNodes online) + { + var serviceSpec = new V1Service + { + ApiVersion = "v1", + Metadata = online.GetServiceMetadata(), + Spec = new V1ServiceSpec + { + Type = "NodePort", + Selector = online.GetSelector(), + Ports = CreateServicePorts(online) + } + }; + + online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); + } + + private List CreateServicePorts(OnlineCodexNodes online) + { + var result = new List(); + var containers = online.GetContainers(); + foreach (var container in containers) + { + result.Add(new V1ServicePort + { + Protocol = "TCP", + Port = 8080, + TargetPort = container.ContainerPortName, + NodePort = container.ServicePort + }); + } + return result; + } + + private void DeleteService(OnlineCodexNodes online) + { + if (online.Service == null) return; + client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); + online.Service = null; + } + + #endregion + + #region Deployment management + + private void CreateDeployment(OnlineCodexNodes online, OfflineCodexNodes offline) + { + var deploymentSpec = new V1Deployment + { + ApiVersion = "apps/v1", + Metadata = online.GetDeploymentMetadata(), + Spec = new V1DeploymentSpec + { + Replicas = 1, + Selector = new V1LabelSelector + { + MatchLabels = online.GetSelector() + }, + Template = new V1PodTemplateSpec + { + Metadata = new V1ObjectMeta + { + Labels = online.GetSelector() + }, + Spec = new V1PodSpec + { + Containers = CreateDeploymentContainers(online, offline) + } + } + } + }; + + online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); + } + + private List CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline) + { + var result = new List(); + var containers = online.GetContainers(); + foreach (var container in containers) + { + result.Add(new V1Container + { + Name = container.Name, + Image = dockerImage.GetImageTag(), + Ports = new List + { + new V1ContainerPort + { + ContainerPort = container.ApiPort, + Name = container.ContainerPortName + } + }, + Env = dockerImage.CreateEnvironmentVariables(offline, container) + }); + } + return result; + } + + private void DeleteDeployment(OnlineCodexNodes online) + { + if (online.Deployment == null) return; + client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace); + online.Deployment = null; + } + + #endregion + + #region Namespace management + + private void EnsureTestNamespace() + { + if (IsTestNamespaceOnline()) return; + + var namespaceSpec = new V1Namespace + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = K8sNamespace, + Labels = new Dictionary { { "name", K8sNamespace } } + } + }; + client.CreateNamespace(namespaceSpec); + } + + private void DeleteNamespace() + { + if (IsTestNamespaceOnline()) + { + client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0); + } + } + + #endregion + + private bool IsTestNamespaceOnline() + { + return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); + } + } +} diff --git a/CodexDistTestCore/KnownK8sPods.cs b/CodexDistTestCore/KnownK8sPods.cs new file mode 100644 index 0000000..940a147 --- /dev/null +++ b/CodexDistTestCore/KnownK8sPods.cs @@ -0,0 +1,17 @@ +namespace CodexDistTestCore +{ + public class KnownK8sPods + { + private readonly List knownActivePodNames = new List(); + + public bool Contains(string name) + { + return knownActivePodNames.Contains(name); + } + + public void Add(string name) + { + knownActivePodNames.Add(name); + } + } +} diff --git a/CodexDistTestCore/OnlineCodexNodes.cs b/CodexDistTestCore/OnlineCodexNodes.cs index 119cf1f..c4007fa 100644 --- a/CodexDistTestCore/OnlineCodexNodes.cs +++ b/CodexDistTestCore/OnlineCodexNodes.cs @@ -28,6 +28,11 @@ namespace CodexDistTestCore } } + public IOfflineCodexNodes BringOffline() + { + return k8SManager.BringOffline(this); + } + public int OrderNumber { get; } public OfflineCodexNodes Origin { get; } public OnlineCodexNode[] Nodes { get; } @@ -35,11 +40,6 @@ namespace CodexDistTestCore public V1Service? Service { get; set; } public List ActivePodNames { get; } = new List(); - public IOfflineCodexNodes BringOffline() - { - return k8SManager.BringOffline(this); - } - public CodexNodeContainer[] GetContainers() { return Nodes.Select(n => n.Container).ToArray(); @@ -50,7 +50,7 @@ namespace CodexDistTestCore return new V1ObjectMeta { Name = "codex-test-entrypoint-" + OrderNumber, - NamespaceProperty = K8sManager.K8sNamespace + NamespaceProperty = K8sOperations.K8sNamespace }; } @@ -59,7 +59,7 @@ namespace CodexDistTestCore return new V1ObjectMeta { Name = "codex-test-node-" + OrderNumber, - NamespaceProperty = K8sManager.K8sNamespace + NamespaceProperty = K8sOperations.K8sNamespace }; } diff --git a/CodexDistTestCore/PodLogsHandler.cs b/CodexDistTestCore/PodLogsHandler.cs new file mode 100644 index 0000000..d7898f5 --- /dev/null +++ b/CodexDistTestCore/PodLogsHandler.cs @@ -0,0 +1,7 @@ +namespace CodexDistTestCore +{ + public interface IPodLogsHandler + { + void Log(int id, string podDescription, Stream log); + } +} diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index e376204..04cad1c 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -8,6 +8,7 @@ namespace CodexDistTestCore private static LogFile? file = null; + // This is all way too static. It needs to be cleaned up. public static void Log(string message) { file!.Write(message); @@ -38,7 +39,8 @@ namespace CodexDistTestCore Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) { - IncludeFullPodLogging(k8sManager); + var logWriter = new PodLogWriter(file); + logWriter.IncludeFullPodLogging(k8sManager); } file = null; @@ -50,24 +52,29 @@ namespace CodexDistTestCore var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); return $"{className}.{test.MethodName}"; } + } - private static void LogRaw(string message, string filename) + public class PodLogWriter : IPodLogsHandler + { + private readonly LogFile file; + + public PodLogWriter(LogFile file) { - file!.WriteRaw(message, filename); + this.file = file; } - private static void IncludeFullPodLogging(K8sManager k8sManager) + public void IncludeFullPodLogging(K8sManager k8sManager) { - Log("Full pod logging:"); - k8sManager.FetchAllPodsLogs(WritePodLog); + TestLog.Log("Full pod logging:"); + k8sManager.FetchAllPodsLogs(this); } - private static void WritePodLog(int id, string nodeDescription, Stream stream) + public void Log(int id, string podDescription, Stream log) { var logFile = id.ToString().PadLeft(6, '0'); - Log($"{nodeDescription} -->> {logFile}"); - LogRaw(nodeDescription, logFile); - var reader = new StreamReader(stream); + TestLog.Log($"{podDescription} -->> {logFile}"); + LogRaw(podDescription, logFile); + var reader = new StreamReader(log); var line = reader.ReadLine(); while (line != null) { @@ -75,6 +82,11 @@ namespace CodexDistTestCore line = reader.ReadLine(); } } + + private void LogRaw(string message, string filename) + { + file!.WriteRaw(message, filename); + } } public class LogFile