From cbf0fbf5b572d684a17a6bc59c888c15f97bf469 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 11:07:36 +0200 Subject: [PATCH] Implements restart test. --- DistTestCore/CodexNodeGroup.cs | 25 ++++++--- DistTestCore/CodexStarter.cs | 15 +++++- KubernetesWorkflow/K8sController.cs | 63 +++++++++++++++++----- KubernetesWorkflow/RunningPod.cs | 6 ++- KubernetesWorkflow/StartupWorkflow.cs | 8 +++ KubernetesWorkflow/WorkflowCreator.cs | 5 +- KubernetesWorkflow/WorkflowNumberSource.cs | 5 +- Tests/BasicTests/SimpleTests.cs | 25 +++++++-- 8 files changed, 121 insertions(+), 31 deletions(-) diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 4ca3a6c..e486162 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -6,7 +6,7 @@ namespace DistTestCore { public interface ICodexNodeGroup : IEnumerable { - //ICodexSetup BringOffline(); + ICodexSetup BringOffline(); IOnlineCodexNode this[int index] { get; } } @@ -30,14 +30,23 @@ namespace DistTestCore } } - //public ICodexSetup BringOffline() - //{ - // //return k8SManager.BringOffline(this); - //} + public ICodexSetup BringOffline() + { + var result = Setup; + var containers = Containers; - public CodexSetup Setup { get; } - public RunningContainers Containers { get; } - public OnlineCodexNode[] Nodes { get; } + // Clear everything. Prevent accidental use. + Setup = null!; + Containers = null!; + Nodes = Array.Empty(); + + lifecycle.CodexStarter.BringOffline(containers); + return result; + } + + public CodexSetup Setup { get; private set; } + public RunningContainers Containers { get; private set; } + public OnlineCodexNode[] Nodes { get; private set; } //public GethCompanionGroup? GethCompanionGroup { get; set; } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 86ed8c3..8ad0aa8 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -16,7 +16,7 @@ namespace DistTestCore public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { - var workflow = workflowCreator.CreateWorkflow(); + var workflow = CreateWorkflow(); var startupConfig = new StartupConfig(); startupConfig.Add(codexSetup); @@ -25,10 +25,21 @@ namespace DistTestCore return new CodexNodeGroup(lifecycle, codexSetup, runningContainers); } + public void BringOffline(RunningContainers runningContainers) + { + var workflow = CreateWorkflow(); + workflow.Stop(runningContainers); + } + public void DeleteAllResources() { - var workflow = workflowCreator.CreateWorkflow(); + var workflow = CreateWorkflow(); workflow.DeleteAllResources(); } + + private StartupWorkflow CreateWorkflow() + { + return workflowCreator.CreateWorkflow(); + } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 295397b..a3b25fa 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -28,11 +28,19 @@ namespace KubernetesWorkflow { EnsureTestNamespace(); - CreateDeployment(containerRecipes, location); - var servicePortsMap = CreateService(containerRecipes); + var deploymentName = CreateDeployment(containerRecipes, location); + var (serviceName, servicePortsMap) = CreateService(containerRecipes); var (podName, podIp) = FetchNewPod(); - return new RunningPod(cluster, podName, podIp, servicePortsMap); + return new RunningPod(cluster, podName, podIp, deploymentName, serviceName, servicePortsMap); + } + + public void Stop(RunningPod pod) + { + if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName); + DeleteDeployment(pod.DeploymentName); + WaitUntilDeploymentOffline(pod.DeploymentName); + WaitUntilPodOffline(pod.Name); } public void DeleteAllResources() @@ -83,7 +91,7 @@ namespace KubernetesWorkflow #region Deployment management - private void CreateDeployment(ContainerRecipe[] containerRecipes, Location location) + private string CreateDeployment(ContainerRecipe[] containerRecipes, Location location) { var deploymentSpec = new V1Deployment { @@ -112,7 +120,15 @@ namespace KubernetesWorkflow }; client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); - WaitUntilDeploymentCreated(deploymentSpec); + WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); + + return deploymentSpec.Metadata.Name; + } + + private void DeleteDeployment(string deploymentName) + { + client.DeleteNamespacedDeployment(deploymentName, K8sNamespace); + WaitUntilDeploymentOffline(deploymentName); } private IDictionary CreateNodeSelector(Location location) @@ -194,7 +210,7 @@ namespace KubernetesWorkflow #region Service management - private Dictionary CreateService(ContainerRecipe[] containerRecipes) + private (string, Dictionary) CreateService(ContainerRecipe[] containerRecipes) { var result = new Dictionary(); @@ -204,7 +220,7 @@ namespace KubernetesWorkflow { // None of these container-recipes wish to expose anything via a serice port. // So, we don't have to create a service. - return result; + return (string.Empty, result); } var serviceSpec = new V1Service @@ -220,7 +236,13 @@ namespace KubernetesWorkflow }; client.CreateNamespacedService(serviceSpec, K8sNamespace); - return result; + + return (serviceSpec.Metadata.Name, result); + } + + private void DeleteService(string serviceName) + { + client.DeleteNamespacedService(serviceName, K8sNamespace); } private V1ObjectMeta CreateServiceMetadata() @@ -279,11 +301,6 @@ namespace KubernetesWorkflow WaitUntil(() => !IsTestNamespaceOnline()); } - private void WaitUntilDeploymentCreated(V1Deployment deploymentSpec) - { - WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); - } - private void WaitUntilDeploymentOnline(string deploymentName) { WaitUntil(() => @@ -293,6 +310,26 @@ namespace KubernetesWorkflow }); } + private void WaitUntilDeploymentOffline(string deploymentName) + { + WaitUntil(() => + { + var deployments = client.ListNamespacedDeployment(K8sNamespace); + var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName); + return deployment == null || deployment.Status.AvailableReplicas == 0; + }); + } + + private void WaitUntilPodOffline(string podName) + { + WaitUntil(() => + { + var pods = client.ListNamespacedPod(K8sNamespace).Items; + var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName); + return pod == null; + }); + } + private void WaitUntil(Func predicate) { var start = DateTime.UtcNow; diff --git a/KubernetesWorkflow/RunningPod.cs b/KubernetesWorkflow/RunningPod.cs index 63378e2..b676903 100644 --- a/KubernetesWorkflow/RunningPod.cs +++ b/KubernetesWorkflow/RunningPod.cs @@ -4,17 +4,21 @@ { private readonly Dictionary servicePortMap; - public RunningPod(K8sCluster cluster, string name, string ip, Dictionary servicePortMap) + public RunningPod(K8sCluster cluster, string name, string ip, string deploymentName, string serviceName, Dictionary servicePortMap) { Cluster = cluster; Name = name; Ip = ip; + DeploymentName = deploymentName; + ServiceName = serviceName; this.servicePortMap = servicePortMap; } public K8sCluster Cluster { get; } public string Name { get; } public string Ip { get; } + internal string DeploymentName { get; } + internal string ServiceName { get; } public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe) { diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 8486590..c3faa50 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -26,6 +26,14 @@ }); } + public void Stop(RunningContainers runningContainers) + { + K8s(controller => + { + controller.Stop(runningContainers.RunningPod); + }); + } + public void DeleteAllResources() { K8s(controller => diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index 88af0e8..51c1c29 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -6,6 +6,7 @@ namespace KubernetesWorkflow { private readonly NumberSource numberSource = new NumberSource(0); private readonly NumberSource servicePortNumberSource = new NumberSource(30001); + private readonly NumberSource containerNumberSource = new NumberSource(0); private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly K8sCluster cluster; @@ -16,7 +17,9 @@ namespace KubernetesWorkflow public StartupWorkflow CreateWorkflow() { - var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), servicePortNumberSource); + var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), + servicePortNumberSource, + containerNumberSource); return new StartupWorkflow(workflowNumberSource, cluster, knownPods); } diff --git a/KubernetesWorkflow/WorkflowNumberSource.cs b/KubernetesWorkflow/WorkflowNumberSource.cs index 018b97b..8cbab34 100644 --- a/KubernetesWorkflow/WorkflowNumberSource.cs +++ b/KubernetesWorkflow/WorkflowNumberSource.cs @@ -4,13 +4,14 @@ namespace KubernetesWorkflow { public class WorkflowNumberSource { - private readonly NumberSource containerNumberSource = new NumberSource(0); private readonly NumberSource servicePortNumberSource; + private readonly NumberSource containerNumberSource; - public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource) + public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource, NumberSource containerNumberSource) { WorkflowNumber = workflowNumber; this.servicePortNumberSource = servicePortNumberSource; + this.containerNumberSource = containerNumberSource; } public int WorkflowNumber { get; } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 4c33425..a723471 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -12,13 +12,19 @@ namespace Tests.BasicTests { var primary = SetupCodexNodes(1).BringOnline()[0]; - var testFile = GenerateTestFile(1.MB()); + PerformOneClientTest(primary); + } - var contentId = primary.UploadFile(testFile); + [Test] + public void RestartTest() + { + var group = SetupCodexNodes(1).BringOnline(); - var downloadedFile = primary.DownloadContent(contentId); + var setup = group.BringOffline(); - testFile.AssertIsEqual(downloadedFile); + var primary = setup.BringOnline()[0]; + + PerformOneClientTest(primary); } [Test] @@ -119,6 +125,17 @@ namespace Tests.BasicTests // //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); //} + private void PerformOneClientTest(IOnlineCodexNode primary) + { + var testFile = GenerateTestFile(1.MB()); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = primary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) { primary.ConnectToPeer(secondary);