setup

DistTestCore/CodexStarter.cs

@@ -44,7 +44,7 @@ namespace DistTestCore
         public void DeleteAllResources()
         {
             var workflow = CreateWorkflow();
-            workflow.DeleteAllResources();
+            workflow.DeleteTestResources();
 
             RunningGroups.Clear();
         }

DistTestCore/Configuration.cs

@@ -7,7 +7,7 @@ namespace DistTestCore
         public KubernetesWorkflow.Configuration GetK8sConfiguration()
         {
             return new KubernetesWorkflow.Configuration(
-                k8sNamespace: "codex-test-ns",
+                k8sNamespacePrefix: "ct-",
                 kubeConfigFile: null,
                 operationTimeout: Timing.K8sOperationTimeout(),
                 retryDelay: Timing.K8sServiceDelay(),

DistTestCore/DistTest.cs

@@ -153,7 +153,8 @@ namespace DistTestCore
 
         private void CreateNewTestLifecycle()
         {
-            Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () =>
+            var testName = GetCurrentTestName();
+            Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
             {
                 lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration);
                 testStart = DateTime.UtcNow;

KubernetesWorkflow/ApplicationLifecycle.cs

@@ -0,0 +1,36 @@
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public class ApplicationLifecycle
+    {
+        private static ApplicationLifecycle? instance;
+        private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
+        private readonly NumberSource namespaceNumberSource = new NumberSource(0);
+
+        private ApplicationLifecycle()
+        {
+        }
+
+        public static ApplicationLifecycle Instance
+        {
+            // I know singletons are quite evil. But we need to be sure this object is created only once
+            // and persists for the entire application lifecycle.
+            get
+            {
+                if (instance == null) instance = new ApplicationLifecycle();
+                return instance;
+            }
+        }
+
+        public NumberSource GetServiceNumberSource()
+        {
+            return servicePortNumberSource;
+        }
+
+        public string GetTestNamespace()
+        {
+            return namespaceNumberSource.GetNextNumber().ToString("D5");
+        }
+    }
+}

KubernetesWorkflow/Configuration.cs

@@ -2,16 +2,16 @@
 {
     public class Configuration
     {
-        public Configuration(string k8sNamespace, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
+        public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
         {
-            K8sNamespace = k8sNamespace;
+            K8sNamespacePrefix = k8sNamespacePrefix;
             KubeConfigFile = kubeConfigFile;
             OperationTimeout = operationTimeout;
             RetryDelay = retryDelay;
             LocationMap = locationMap;
         }
 
-        public string K8sNamespace { get; }
+        public string K8sNamespacePrefix { get; }
         public string? KubeConfigFile { get; }
         public TimeSpan OperationTimeout { get; }
         public TimeSpan RetryDelay { get; }

KubernetesWorkflow/K8sController.cs

@@ -11,15 +11,16 @@ namespace KubernetesWorkflow
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownPods;
         private readonly WorkflowNumberSource workflowNumberSource;
+        private readonly string testNamespace;
         private readonly Kubernetes client;
 
-        public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource)
+        public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
         {
             this.log = log;
             this.cluster = cluster;
             this.knownPods = knownPods;
             this.workflowNumberSource = workflowNumberSource;
-
+            this.testNamespace = testNamespace;
             client = new Kubernetes(cluster.GetK8sClientConfig());
         }
 
@@ -52,14 +53,14 @@ namespace KubernetesWorkflow
         public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
         {
             log.Debug();
-            using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name);
+            using var stream = client.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name);
             logHandler.Log(stream);
         }
 
         public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
         {
             log.Debug($"{containerName}: {command} ({string.Join(",", args)})");
-            var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
+            var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args);
             runner.Run();
             return runner.GetStdOut();
         }
@@ -67,11 +68,39 @@ namespace KubernetesWorkflow
         public void DeleteAllResources()
         {
             log.Debug();
-            DeleteNamespace();
 
+            var all = client.ListNamespace().Items;
+            var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix));
+
+            foreach (var ns in namespaces)
+            {
+                DeleteNamespace(ns);
+            }
+            foreach (var ns in namespaces)
+            {
+                WaitUntilNamespaceDeleted(ns);
+            }
+        }
+
+        public void DeleteTestNamespace()
+        {
+            log.Debug();
+            if (IsTestNamespaceOnline())
+            {
+                client.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0);
+            }
             WaitUntilNamespaceDeleted();
         }
 
+        public void DeleteNamespace(string ns)
+        {
+            log.Debug();
+            if (IsNamespaceOnline(ns))
+            {
+                client.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0);
+            }
+        }
+
         #region Namespace management
 
         private void EnsureTestNamespace()
@@ -83,30 +112,27 @@ namespace KubernetesWorkflow
                 ApiVersion = "v1",
                 Metadata = new V1ObjectMeta
                 {
-                    Name = K8sNamespace,
+                    Name = K8sTestNamespace,
-                    Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
+                    Labels = new Dictionary<string, string> { { "name", K8sTestNamespace } }
                 }
             };
             client.CreateNamespace(namespaceSpec);
             WaitUntilNamespaceCreated();
         }
 
-        private void DeleteNamespace()
+        private string K8sTestNamespace
         {
-            if (IsTestNamespaceOnline())
+            get { return cluster.Configuration.K8sNamespacePrefix + testNamespace; }
-            {
-                client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
-            }
-        }
-
-        private string K8sNamespace
-        {
-            get { return cluster.Configuration.K8sNamespace; }
         }
 
         private bool IsTestNamespaceOnline()
         {
-            return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
+            return IsNamespaceOnline(K8sTestNamespace);
+        }
+
+        private bool IsNamespaceOnline(string name)
+        {
+            return client.ListNamespace().Items.Any(n => n.Metadata.Name == name);
         }
 
         #endregion
@@ -141,7 +167,7 @@ namespace KubernetesWorkflow
                 }
             };
 
-            client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
+            client.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace);
             WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
 
             return deploymentSpec.Metadata.Name;
@@ -149,7 +175,7 @@ namespace KubernetesWorkflow
 
         private void DeleteDeployment(string deploymentName)
         {
-            client.DeleteNamespacedDeployment(deploymentName, K8sNamespace);
+            client.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace);
             WaitUntilDeploymentOffline(deploymentName);
         }
 
@@ -173,7 +199,7 @@ namespace KubernetesWorkflow
             return new V1ObjectMeta
             {
                 Name = "deploy-" + workflowNumberSource.WorkflowNumber,
-                NamespaceProperty = K8sNamespace
+                NamespaceProperty = K8sTestNamespace
             };
         }
 
@@ -257,14 +283,14 @@ namespace KubernetesWorkflow
                 }
             };
 
-            client.CreateNamespacedService(serviceSpec, K8sNamespace);
+            client.CreateNamespacedService(serviceSpec, K8sTestNamespace);
 
             return (serviceSpec.Metadata.Name, result);
         }
 
         private void DeleteService(string serviceName)
         {
-            client.DeleteNamespacedService(serviceName, K8sNamespace);
+            client.DeleteNamespacedService(serviceName, K8sTestNamespace);
         }
 
         private V1ObjectMeta CreateServiceMetadata()
@@ -272,7 +298,7 @@ namespace KubernetesWorkflow
             return new V1ObjectMeta
             {
                 Name = "service-" + workflowNumberSource.WorkflowNumber,
-                NamespaceProperty = K8sNamespace
+                NamespaceProperty = K8sTestNamespace
             };
         }
 
@@ -323,11 +349,16 @@ namespace KubernetesWorkflow
             WaitUntil(() => !IsTestNamespaceOnline());
         }
 
+        private void WaitUntilNamespaceDeleted(string name)
+        {
+            WaitUntil(() => !IsNamespaceOnline(name));
+        }
+
         private void WaitUntilDeploymentOnline(string deploymentName)
         {
             WaitUntil(() =>
             {
-                var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
+                var deployment = client.ReadNamespacedDeployment(deploymentName, K8sTestNamespace);
                 return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
             });
         }
@@ -336,7 +367,7 @@ namespace KubernetesWorkflow
         {
             WaitUntil(() =>
             {
-                var deployments = client.ListNamespacedDeployment(K8sNamespace);
+                var deployments = client.ListNamespacedDeployment(K8sTestNamespace);
                 var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
                 return deployment == null || deployment.Status.AvailableReplicas == 0;
             });
@@ -346,7 +377,7 @@ namespace KubernetesWorkflow
         {
             WaitUntil(() =>
             {
-                var pods = client.ListNamespacedPod(K8sNamespace).Items;
+                var pods = client.ListNamespacedPod(K8sTestNamespace).Items;
                 var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
                 return pod == null;
             });
@@ -369,7 +400,7 @@ namespace KubernetesWorkflow
 
         private (string, string) FetchNewPod()
         {
-            var pods = client.ListNamespacedPod(K8sNamespace).Items;
+            var pods = client.ListNamespacedPod(K8sTestNamespace).Items;
 
             var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
             if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");

KubernetesWorkflow/StartupWorkflow.cs

@@ -8,14 +8,16 @@ namespace KubernetesWorkflow
         private readonly WorkflowNumberSource numberSource;
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownK8SPods;
+        private readonly string testNamespace;
         private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
 
-        internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods)
+        internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
         {
             this.log = log;
             this.numberSource = numberSource;
             this.cluster = cluster;
             this.knownK8SPods = knownK8SPods;
+            this.testNamespace = testNamespace;
         }
 
         public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
@@ -62,6 +64,14 @@ namespace KubernetesWorkflow
             });
         }
 
+        public void DeleteTestResources()
+        {
+            K8s(controller =>
+            {
+                controller.DeleteTestNamespace();
+            });
+        }
+
         private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig)
         {
             log.Debug();
@@ -82,14 +92,14 @@ namespace KubernetesWorkflow
 
         private void K8s(Action<K8sController> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
             action(controller);
             controller.Dispose();
         }
 
         private T K8s<T>(Func<K8sController, T> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
             var result = action(controller);
             controller.Dispose();
             return result;

KubernetesWorkflow/WorkflowCreator.cs

@@ -6,7 +6,6 @@ namespace KubernetesWorkflow
     public class WorkflowCreator
     {
         private readonly NumberSource numberSource = new NumberSource(0);
-        private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
         private readonly NumberSource containerNumberSource = new NumberSource(0);
         private readonly KnownK8sPods knownPods = new KnownK8sPods();
         private readonly K8sCluster cluster;
@@ -21,10 +20,10 @@ namespace KubernetesWorkflow
         public StartupWorkflow CreateWorkflow()
         {
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
-                                                                    servicePortNumberSource,
+                                                                    ApplicationLifecycle.Instance.GetServiceNumberSource(),
                                                                     containerNumberSource);
 
-            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods);
+            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, ApplicationLifecycle.Instance.GetTestNamespace());
         }
     }
 }

Tests/BasicTests/TwoClientTests.cs

@@ -5,6 +5,7 @@ using NUnit.Framework;
 namespace Tests.BasicTests
 {
     [TestFixture]
+    [Parallelizable(ParallelScope.All)]
     public class TwoClientTests : DistTest
     {
         [Test]