Implements restart test.

DistTestCore/CodexNodeGroup.cs

@@ -6,7 +6,7 @@ namespace DistTestCore
 {
     public interface ICodexNodeGroup : IEnumerable<IOnlineCodexNode>
     {
-        //ICodexSetup BringOffline();
+        ICodexSetup BringOffline();
         IOnlineCodexNode this[int index] { get; }
     }
 
@@ -30,14 +30,23 @@ namespace DistTestCore
             }
         }
 
-        //public ICodexSetup BringOffline()
+        public ICodexSetup BringOffline()
-        //{
+        {
-        //    //return k8SManager.BringOffline(this);
+            var result = Setup;
-        //}
+            var containers = Containers;
 
-        public CodexSetup Setup { get; }
+            // Clear everything. Prevent accidental use.
-        public RunningContainers Containers { get; }
+            Setup = null!;
-        public OnlineCodexNode[] Nodes { get; }
+            Containers = null!;
+            Nodes = Array.Empty<OnlineCodexNode>();
+
+            lifecycle.CodexStarter.BringOffline(containers);
+            return result;
+        }
+
+        public CodexSetup Setup { get; private set; }
+        public RunningContainers Containers { get; private set; }
+        public OnlineCodexNode[] Nodes { get; private set; }
 
         //public GethCompanionGroup? GethCompanionGroup { get; set; }
 

DistTestCore/CodexStarter.cs

@@ -16,7 +16,7 @@ namespace DistTestCore
 
         public ICodexNodeGroup BringOnline(CodexSetup codexSetup)
         {
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = CreateWorkflow();
             var startupConfig = new StartupConfig();
             startupConfig.Add(codexSetup);
             
@@ -25,10 +25,21 @@ namespace DistTestCore
             return new CodexNodeGroup(lifecycle, codexSetup, runningContainers);
         }
 
+        public void BringOffline(RunningContainers runningContainers)
+        {
+            var workflow = CreateWorkflow();
+            workflow.Stop(runningContainers);
+        }
+
         public void DeleteAllResources()
         {
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = CreateWorkflow();
             workflow.DeleteAllResources();
         }
+
+        private StartupWorkflow CreateWorkflow()
+        {
+            return workflowCreator.CreateWorkflow();
+        }
     }
 }

KubernetesWorkflow/K8sController.cs

@@ -28,11 +28,19 @@ namespace KubernetesWorkflow
         {
             EnsureTestNamespace();
 
-            CreateDeployment(containerRecipes, location);
+            var deploymentName = CreateDeployment(containerRecipes, location);
-            var servicePortsMap = CreateService(containerRecipes);
+            var (serviceName, servicePortsMap) = CreateService(containerRecipes);
             var (podName, podIp) = FetchNewPod();
 
-            return new RunningPod(cluster, podName, podIp, servicePortsMap);
+            return new RunningPod(cluster, podName, podIp, deploymentName, serviceName, servicePortsMap);
+        }
+
+        public void Stop(RunningPod pod)
+        {
+            if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName);
+            DeleteDeployment(pod.DeploymentName);
+            WaitUntilDeploymentOffline(pod.DeploymentName);
+            WaitUntilPodOffline(pod.Name);
         }
 
         public void DeleteAllResources()
@@ -83,7 +91,7 @@ namespace KubernetesWorkflow
 
         #region Deployment management
 
-        private void CreateDeployment(ContainerRecipe[] containerRecipes, Location location)
+        private string CreateDeployment(ContainerRecipe[] containerRecipes, Location location)
         {
             var deploymentSpec = new V1Deployment
             {
@@ -112,7 +120,15 @@ namespace KubernetesWorkflow
             };
 
             client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
-            WaitUntilDeploymentCreated(deploymentSpec);
+            WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
+
+            return deploymentSpec.Metadata.Name;
+        }
+
+        private void DeleteDeployment(string deploymentName)
+        {
+            client.DeleteNamespacedDeployment(deploymentName, K8sNamespace);
+            WaitUntilDeploymentOffline(deploymentName);
         }
 
         private IDictionary<string, string> CreateNodeSelector(Location location)
@@ -194,7 +210,7 @@ namespace KubernetesWorkflow
 
         #region Service management
 
-        private Dictionary<ContainerRecipe, Port[]> CreateService(ContainerRecipe[] containerRecipes)
+        private (string, Dictionary<ContainerRecipe, Port[]>) CreateService(ContainerRecipe[] containerRecipes)
         {
             var result = new Dictionary<ContainerRecipe, Port[]>();
 
@@ -204,7 +220,7 @@ namespace KubernetesWorkflow
             {
                 // None of these container-recipes wish to expose anything via a serice port.
                 // So, we don't have to create a service.
-                return result;
+                return (string.Empty, result);
             }
 
             var serviceSpec = new V1Service
@@ -220,7 +236,13 @@ namespace KubernetesWorkflow
             };
 
             client.CreateNamespacedService(serviceSpec, K8sNamespace);
-            return result;
+
+            return (serviceSpec.Metadata.Name, result);
+        }
+
+        private void DeleteService(string serviceName)
+        {
+            client.DeleteNamespacedService(serviceName, K8sNamespace);
         }
 
         private V1ObjectMeta CreateServiceMetadata()
@@ -279,11 +301,6 @@ namespace KubernetesWorkflow
             WaitUntil(() => !IsTestNamespaceOnline());
         }
 
-        private void WaitUntilDeploymentCreated(V1Deployment deploymentSpec)
-        {
-            WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
-        }
-
         private void WaitUntilDeploymentOnline(string deploymentName)
         {
             WaitUntil(() =>
@@ -293,6 +310,26 @@ namespace KubernetesWorkflow
             });
         }
 
+        private void WaitUntilDeploymentOffline(string deploymentName)
+        {
+            WaitUntil(() =>
+            {
+                var deployments = client.ListNamespacedDeployment(K8sNamespace);
+                var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
+                return deployment == null || deployment.Status.AvailableReplicas == 0;
+            });
+        }
+
+        private void WaitUntilPodOffline(string podName)
+        {
+            WaitUntil(() =>
+            {
+                var pods = client.ListNamespacedPod(K8sNamespace).Items;
+                var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
+                return pod == null;
+            });
+        }
+
         private void WaitUntil(Func<bool> predicate)
         {
             var start = DateTime.UtcNow;

KubernetesWorkflow/RunningPod.cs

@@ -4,17 +4,21 @@
     {
         private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap;
 
-        public RunningPod(K8sCluster cluster, string name, string ip, Dictionary<ContainerRecipe, Port[]> servicePortMap)
+        public RunningPod(K8sCluster cluster, string name, string ip, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
         {
             Cluster = cluster;
             Name = name;
             Ip = ip;
+            DeploymentName = deploymentName;
+            ServiceName = serviceName;
             this.servicePortMap = servicePortMap;
         }
 
         public K8sCluster Cluster { get; }
         public string Name { get; }
         public string Ip { get; }
+        internal string DeploymentName { get; }
+        internal string ServiceName { get; }
 
         public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
         {

KubernetesWorkflow/StartupWorkflow.cs

@@ -26,6 +26,14 @@
             });
         }
 
+        public void Stop(RunningContainers runningContainers)
+        {
+            K8s(controller =>
+            {
+                controller.Stop(runningContainers.RunningPod);
+            });
+        }
+
         public void DeleteAllResources()
         {
             K8s(controller =>

KubernetesWorkflow/WorkflowCreator.cs

@@ -6,6 +6,7 @@ namespace KubernetesWorkflow
     {
         private readonly NumberSource numberSource = new NumberSource(0);
         private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
+        private readonly NumberSource containerNumberSource = new NumberSource(0);
         private readonly KnownK8sPods knownPods = new KnownK8sPods();
         private readonly K8sCluster cluster;
 
@@ -16,7 +17,9 @@ namespace KubernetesWorkflow
 
         public StartupWorkflow CreateWorkflow()
         {
-            var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), servicePortNumberSource);
+            var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
+                                                                    servicePortNumberSource,
+                                                                    containerNumberSource);
 
             return new StartupWorkflow(workflowNumberSource, cluster, knownPods);
         }

KubernetesWorkflow/WorkflowNumberSource.cs

@@ -4,13 +4,14 @@ namespace KubernetesWorkflow
 {
     public class WorkflowNumberSource
     {
-        private readonly NumberSource containerNumberSource = new NumberSource(0);
         private readonly NumberSource servicePortNumberSource;
+        private readonly NumberSource containerNumberSource;
 
-        public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource)
+        public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource, NumberSource containerNumberSource)
         {
             WorkflowNumber = workflowNumber;
             this.servicePortNumberSource = servicePortNumberSource;
+            this.containerNumberSource = containerNumberSource;
         }
 
         public int WorkflowNumber { get; }

Tests/BasicTests/SimpleTests.cs

@@ -12,13 +12,19 @@ namespace Tests.BasicTests
         {
             var primary = SetupCodexNodes(1).BringOnline()[0];
 
-            var testFile = GenerateTestFile(1.MB());
+            PerformOneClientTest(primary);
+        }
 
-            var contentId = primary.UploadFile(testFile);
+        [Test]
+        public void RestartTest()
+        {
+            var group = SetupCodexNodes(1).BringOnline();
 
-            var downloadedFile = primary.DownloadContent(contentId);
+            var setup = group.BringOffline();
 
-            testFile.AssertIsEqual(downloadedFile);
+            var primary = setup.BringOnline()[0];
+
+            PerformOneClientTest(primary);
         }
 
         [Test]
@@ -119,6 +125,17 @@ namespace Tests.BasicTests
         //    //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage.");
         //}
 
+        private void PerformOneClientTest(IOnlineCodexNode primary)
+        {
+            var testFile = GenerateTestFile(1.MB());
+
+            var contentId = primary.UploadFile(testFile);
+
+            var downloadedFile = primary.DownloadContent(contentId);
+
+            testFile.AssertIsEqual(downloadedFile);
+        }
+
         private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
         {
             primary.ConnectToPeer(secondary);