diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 900967a9..52c70ca6 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -44,7 +44,7 @@ namespace DistTestCore public void DeleteAllResources() { var workflow = CreateWorkflow(); - workflow.DeleteAllResources(); + workflow.DeleteTestResources(); RunningGroups.Clear(); } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index 5de9afcd..b60a11f7 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -7,7 +7,7 @@ namespace DistTestCore public KubernetesWorkflow.Configuration GetK8sConfiguration() { return new KubernetesWorkflow.Configuration( - k8sNamespace: "codex-test-ns", + k8sNamespacePrefix: "ct-", kubeConfigFile: null, operationTimeout: Timing.K8sOperationTimeout(), retryDelay: Timing.K8sServiceDelay(), diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index a85f689d..b0686e15 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -153,7 +153,8 @@ namespace DistTestCore private void CreateNewTestLifecycle() { - Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () => + var testName = GetCurrentTestName(); + Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () => { lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration); testStart = DateTime.UtcNow; diff --git a/KubernetesWorkflow/ApplicationLifecycle.cs b/KubernetesWorkflow/ApplicationLifecycle.cs new file mode 100644 index 00000000..5289573f --- /dev/null +++ b/KubernetesWorkflow/ApplicationLifecycle.cs @@ -0,0 +1,36 @@ +using Utils; + +namespace KubernetesWorkflow +{ + public class ApplicationLifecycle + { + private static ApplicationLifecycle? instance; + private readonly NumberSource servicePortNumberSource = new NumberSource(30001); + private readonly NumberSource namespaceNumberSource = new NumberSource(0); + + private ApplicationLifecycle() + { + } + + public static ApplicationLifecycle Instance + { + // I know singletons are quite evil. But we need to be sure this object is created only once + // and persists for the entire application lifecycle. + get + { + if (instance == null) instance = new ApplicationLifecycle(); + return instance; + } + } + + public NumberSource GetServiceNumberSource() + { + return servicePortNumberSource; + } + + public string GetTestNamespace() + { + return namespaceNumberSource.GetNextNumber().ToString("D5"); + } + } +} diff --git a/KubernetesWorkflow/Configuration.cs b/KubernetesWorkflow/Configuration.cs index b5a4779c..f94924d3 100644 --- a/KubernetesWorkflow/Configuration.cs +++ b/KubernetesWorkflow/Configuration.cs @@ -2,16 +2,16 @@ { public class Configuration { - public Configuration(string k8sNamespace, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap) + public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap) { - K8sNamespace = k8sNamespace; + K8sNamespacePrefix = k8sNamespacePrefix; KubeConfigFile = kubeConfigFile; OperationTimeout = operationTimeout; RetryDelay = retryDelay; LocationMap = locationMap; } - public string K8sNamespace { get; } + public string K8sNamespacePrefix { get; } public string? KubeConfigFile { get; } public TimeSpan OperationTimeout { get; } public TimeSpan RetryDelay { get; } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 21e82661..118101d8 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -11,15 +11,16 @@ namespace KubernetesWorkflow private readonly K8sCluster cluster; private readonly KnownK8sPods knownPods; private readonly WorkflowNumberSource workflowNumberSource; + private readonly string testNamespace; private readonly Kubernetes client; - public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource) + public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace) { this.log = log; this.cluster = cluster; this.knownPods = knownPods; this.workflowNumberSource = workflowNumberSource; - + this.testNamespace = testNamespace; client = new Kubernetes(cluster.GetK8sClientConfig()); } @@ -52,14 +53,14 @@ namespace KubernetesWorkflow public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) { log.Debug(); - using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name); + using var stream = client.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name); logHandler.Log(stream); } public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args) { log.Debug($"{containerName}: {command} ({string.Join(",", args)})"); - var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args); + var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args); runner.Run(); return runner.GetStdOut(); } @@ -67,11 +68,39 @@ namespace KubernetesWorkflow public void DeleteAllResources() { log.Debug(); - DeleteNamespace(); + var all = client.ListNamespace().Items; + var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix)); + + foreach (var ns in namespaces) + { + DeleteNamespace(ns); + } + foreach (var ns in namespaces) + { + WaitUntilNamespaceDeleted(ns); + } + } + + public void DeleteTestNamespace() + { + log.Debug(); + if (IsTestNamespaceOnline()) + { + client.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0); + } WaitUntilNamespaceDeleted(); } + public void DeleteNamespace(string ns) + { + log.Debug(); + if (IsNamespaceOnline(ns)) + { + client.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0); + } + } + #region Namespace management private void EnsureTestNamespace() @@ -83,30 +112,27 @@ namespace KubernetesWorkflow ApiVersion = "v1", Metadata = new V1ObjectMeta { - Name = K8sNamespace, - Labels = new Dictionary { { "name", K8sNamespace } } + Name = K8sTestNamespace, + Labels = new Dictionary { { "name", K8sTestNamespace } } } }; client.CreateNamespace(namespaceSpec); WaitUntilNamespaceCreated(); } - private void DeleteNamespace() + private string K8sTestNamespace { - if (IsTestNamespaceOnline()) - { - client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0); - } - } - - private string K8sNamespace - { - get { return cluster.Configuration.K8sNamespace; } + get { return cluster.Configuration.K8sNamespacePrefix + testNamespace; } } private bool IsTestNamespaceOnline() { - return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); + return IsNamespaceOnline(K8sTestNamespace); + } + + private bool IsNamespaceOnline(string name) + { + return client.ListNamespace().Items.Any(n => n.Metadata.Name == name); } #endregion @@ -141,7 +167,7 @@ namespace KubernetesWorkflow } }; - client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); + client.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace); WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); return deploymentSpec.Metadata.Name; @@ -149,7 +175,7 @@ namespace KubernetesWorkflow private void DeleteDeployment(string deploymentName) { - client.DeleteNamespacedDeployment(deploymentName, K8sNamespace); + client.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace); WaitUntilDeploymentOffline(deploymentName); } @@ -173,7 +199,7 @@ namespace KubernetesWorkflow return new V1ObjectMeta { Name = "deploy-" + workflowNumberSource.WorkflowNumber, - NamespaceProperty = K8sNamespace + NamespaceProperty = K8sTestNamespace }; } @@ -257,14 +283,14 @@ namespace KubernetesWorkflow } }; - client.CreateNamespacedService(serviceSpec, K8sNamespace); + client.CreateNamespacedService(serviceSpec, K8sTestNamespace); return (serviceSpec.Metadata.Name, result); } private void DeleteService(string serviceName) { - client.DeleteNamespacedService(serviceName, K8sNamespace); + client.DeleteNamespacedService(serviceName, K8sTestNamespace); } private V1ObjectMeta CreateServiceMetadata() @@ -272,7 +298,7 @@ namespace KubernetesWorkflow return new V1ObjectMeta { Name = "service-" + workflowNumberSource.WorkflowNumber, - NamespaceProperty = K8sNamespace + NamespaceProperty = K8sTestNamespace }; } @@ -323,11 +349,16 @@ namespace KubernetesWorkflow WaitUntil(() => !IsTestNamespaceOnline()); } + private void WaitUntilNamespaceDeleted(string name) + { + WaitUntil(() => !IsNamespaceOnline(name)); + } + private void WaitUntilDeploymentOnline(string deploymentName) { WaitUntil(() => { - var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); + var deployment = client.ReadNamespacedDeployment(deploymentName, K8sTestNamespace); return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0; }); } @@ -336,7 +367,7 @@ namespace KubernetesWorkflow { WaitUntil(() => { - var deployments = client.ListNamespacedDeployment(K8sNamespace); + var deployments = client.ListNamespacedDeployment(K8sTestNamespace); var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName); return deployment == null || deployment.Status.AvailableReplicas == 0; }); @@ -346,7 +377,7 @@ namespace KubernetesWorkflow { WaitUntil(() => { - var pods = client.ListNamespacedPod(K8sNamespace).Items; + var pods = client.ListNamespacedPod(K8sTestNamespace).Items; var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName); return pod == null; }); @@ -369,7 +400,7 @@ namespace KubernetesWorkflow private (string, string) FetchNewPod() { - var pods = client.ListNamespacedPod(K8sNamespace).Items; + var pods = client.ListNamespacedPod(K8sTestNamespace).Items; var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure."); diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 8aaba75e..cd229b07 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -8,14 +8,16 @@ namespace KubernetesWorkflow private readonly WorkflowNumberSource numberSource; private readonly K8sCluster cluster; private readonly KnownK8sPods knownK8SPods; + private readonly string testNamespace; private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); - internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods) + internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace) { this.log = log; this.numberSource = numberSource; this.cluster = cluster; this.knownK8SPods = knownK8SPods; + this.testNamespace = testNamespace; } public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) @@ -62,6 +64,14 @@ namespace KubernetesWorkflow }); } + public void DeleteTestResources() + { + K8s(controller => + { + controller.DeleteTestNamespace(); + }); + } + private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig) { log.Debug(); @@ -82,14 +92,14 @@ namespace KubernetesWorkflow private void K8s(Action action) { - var controller = new K8sController(log, cluster, knownK8SPods, numberSource); + var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace); action(controller); controller.Dispose(); } private T K8s(Func action) { - var controller = new K8sController(log, cluster, knownK8SPods, numberSource); + var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace); var result = action(controller); controller.Dispose(); return result; diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index 1f54beae..e4f8032c 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -6,7 +6,6 @@ namespace KubernetesWorkflow public class WorkflowCreator { private readonly NumberSource numberSource = new NumberSource(0); - private readonly NumberSource servicePortNumberSource = new NumberSource(30001); private readonly NumberSource containerNumberSource = new NumberSource(0); private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly K8sCluster cluster; @@ -21,10 +20,10 @@ namespace KubernetesWorkflow public StartupWorkflow CreateWorkflow() { var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), - servicePortNumberSource, + ApplicationLifecycle.Instance.GetServiceNumberSource(), containerNumberSource); - return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods); + return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, ApplicationLifecycle.Instance.GetTestNamespace()); } } } diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs index 90af450d..14e0e215 100644 --- a/Tests/BasicTests/TwoClientTests.cs +++ b/Tests/BasicTests/TwoClientTests.cs @@ -5,6 +5,7 @@ using NUnit.Framework; namespace Tests.BasicTests { [TestFixture] + [Parallelizable(ParallelScope.All)] public class TwoClientTests : DistTest { [Test]