From e87f255f482f3bcff3308546aa0b2e504e64d01f Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 31 Oct 2023 11:01:10 +0100 Subject: [PATCH 01/10] Creates dockerimage that will deploy-and-run from environment args --- Framework/Core/Http.cs | 2 +- Tests/CodexContinuousTests/deploy-and-run.sh | 9 +++++---- docker/deployandrun.Dockerfile | 12 ++++++++++++ docker/docker-compose.yaml | 9 +++++++++ docker/docker-dnr-entrypoint.sh | 5 +++++ 5 files changed, 32 insertions(+), 5 deletions(-) create mode 100644 docker/deployandrun.Dockerfile create mode 100644 docker/docker-dnr-entrypoint.sh diff --git a/Framework/Core/Http.cs b/Framework/Core/Http.cs index 968bdb78..192152f4 100644 --- a/Framework/Core/Http.cs +++ b/Framework/Core/Http.cs @@ -93,7 +93,7 @@ namespace Core { var response = PostJsonString(route, body); if (response == null) throw new Exception("Received no response."); - var result = JsonConvert.DeserializeObject(response); + var result = Deserialize(response); if (result == null) throw new Exception("Failed to deserialize response"); return result; }, $"HTTO-POST-JSON: {route}"); diff --git a/Tests/CodexContinuousTests/deploy-and-run.sh b/Tests/CodexContinuousTests/deploy-and-run.sh index fdd620cc..5dd7a5ea 100644 --- a/Tests/CodexContinuousTests/deploy-and-run.sh +++ b/Tests/CodexContinuousTests/deploy-and-run.sh @@ -1,8 +1,9 @@ set -e -replication=5 -name=testnamehere -filter=TwoClient +replication=$DNR_REP +name=$DNR_NAME +filter=$DNR_FILTER +duration=$DNR_DURATION echo "Deploying..." cd ../../Tools/CodexNetDeployer @@ -45,7 +46,7 @@ do --filter=$filter \ --cleanup=1 \ --full-container-logs=1 \ - --target-duration=172800 # 48 hours + --target-duration=$duration sleep 30 done diff --git a/docker/deployandrun.Dockerfile b/docker/deployandrun.Dockerfile new file mode 100644 index 00000000..869f0178 --- /dev/null +++ b/docker/deployandrun.Dockerfile @@ -0,0 +1,12 @@ +FROM mcr.microsoft.com/dotnet/sdk:7.0 + +RUN apt-get update && apt-get install -y screen +WORKDIR /app +COPY --chmod=0755 docker/docker-dnr-entrypoint.sh / +COPY ./Tools ./Tools +COPY ./Tests ./Tests +COPY ./Framework ./Framework +COPY ./ProjectPlugins ./ProjectPlugins + +ENTRYPOINT ["/docker-dnr-entrypoint.sh"] +CMD ["/bin/bash", "deploy-and-run.sh"] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 98af2465..510f654d 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -9,3 +9,12 @@ services: - KUBECONFIG=/opt/kubeconfig - LOGPATH=/opt/logs - RUNNERLOCATION=ExternalToCluster + + continuous-test-run: + image: thatbenbierens/dist-tests-deployandrun:initial + environment: + # - CODEXDOCKERIMAGE=imageoverride + - DNR_REP=3 + - DNR_NAME=Tryout + - DNR_FILTER=PeernBeer + - DNR_DURATION=172800 diff --git a/docker/docker-dnr-entrypoint.sh b/docker/docker-dnr-entrypoint.sh new file mode 100644 index 00000000..5475acb3 --- /dev/null +++ b/docker/docker-dnr-entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/bash +echo "Running continuous tests..." +cd /app/Tests/CodexContinuousTests +exec "$@" + From bfdbebb36e568898c91f545b9d83b55b8884c133 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 31 Oct 2023 11:15:08 +0100 Subject: [PATCH 02/10] Adds deploy-and-run plugin --- .../CoreInterfaceExtensions.cs | 13 ++++++ .../DeployAndRunContainerRecipe.cs | 43 +++++++++++++++++++ .../DeployAndRunPlugin/DeployAndRunPlugin.cs | 35 +++++++++++++++ .../DeployAndRunPlugin.csproj | 14 ++++++ 4 files changed, 105 insertions(+) create mode 100644 ProjectPlugins/DeployAndRunPlugin/CoreInterfaceExtensions.cs create mode 100644 ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs create mode 100644 ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs create mode 100644 ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.csproj diff --git a/ProjectPlugins/DeployAndRunPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/DeployAndRunPlugin/CoreInterfaceExtensions.cs new file mode 100644 index 00000000..262e27d6 --- /dev/null +++ b/ProjectPlugins/DeployAndRunPlugin/CoreInterfaceExtensions.cs @@ -0,0 +1,13 @@ +using Core; +using KubernetesWorkflow; + +namespace DeployAndRunPlugin +{ + public static class CoreInterfaceExtensions + { + public static RunningContainer DeployAndRunContinuousTests(this CoreInterface ci, RunConfig runConfig) + { + return ci.GetPlugin().Run(runConfig); + } + } +} diff --git a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs new file mode 100644 index 00000000..12435a88 --- /dev/null +++ b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs @@ -0,0 +1,43 @@ +using KubernetesWorkflow; + +namespace DeployAndRunPlugin +{ + public class DeployAndRunContainerRecipe : ContainerRecipeFactory + { + public override string AppName => "deploy-and-run"; + public override string Image => "thatbenbierens/dist-tests-deployandrun:initial"; + + protected override void Initialize(StartupConfig config) + { + var setup = config.Get(); + + if (setup.CodexImageOverride != null) + { + AddEnvVar("CODEXDOCKERIMAGE", setup.CodexImageOverride); + } + + AddEnvVar("DNR_REP", setup.Replications.ToString()); + AddEnvVar("DNR_NAME", setup.Name); + AddEnvVar("DNR_FILTER", setup.Filter); + AddEnvVar("DNR_DURATION", setup.Duration.TotalSeconds.ToString()); + } + } + + public class RunConfig + { + public RunConfig(string name, string filter, TimeSpan duration, int replications, string? codexImageOverride = null) + { + Name = name; + Filter = filter; + Duration = duration; + Replications = replications; + CodexImageOverride = codexImageOverride; + } + + public string Name { get; } + public string Filter { get; } + public TimeSpan Duration { get; } + public int Replications { get; } + public string? CodexImageOverride { get; } + } +} \ No newline at end of file diff --git a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs new file mode 100644 index 00000000..14405443 --- /dev/null +++ b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs @@ -0,0 +1,35 @@ +using Core; +using KubernetesWorkflow; + +namespace DeployAndRunPlugin +{ + public class DeployAndRunPlugin : IProjectPlugin + { + private readonly IPluginTools tools; + + public DeployAndRunPlugin(IPluginTools tools) + { + this.tools = tools; + } + + public void Announce() + { + tools.GetLog().Log("Deploy-and-Run plugin loaded."); + } + + public void Decommission() + { + } + + public RunningContainer Run(RunConfig config) + { + var workflow = tools.CreateWorkflow(); + var startupConfig = new StartupConfig(); + startupConfig.NameOverride = "dnr-" + config.Name; + startupConfig.Add(config); + + var containers = workflow.Start(1, new DeployAndRunContainerRecipe(), startupConfig); + return containers.Containers.Single(); + } + } +} diff --git a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.csproj b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.csproj new file mode 100644 index 00000000..66d0737a --- /dev/null +++ b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.csproj @@ -0,0 +1,14 @@ + + + + net7.0 + enable + enable + + + + + + + + From c348ca98490e4d4d84e3251a7e8eeafa0b7c490d Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 31 Oct 2023 11:22:59 +0100 Subject: [PATCH 03/10] sets up test starter tool --- Tools/TestClusterStarter/ClusterTestSpec.cs | 42 ++++++++++++++++ Tools/TestClusterStarter/Configuration.cs | 10 ++++ Tools/TestClusterStarter/Program.cs | 50 +++++++++++++++++++ .../TestClusterStarter.csproj | 21 ++++++++ cs-codex-dist-testing.sln | 14 ++++++ docker/build-deployandrun.bat | 2 + 6 files changed, 139 insertions(+) create mode 100644 Tools/TestClusterStarter/ClusterTestSpec.cs create mode 100644 Tools/TestClusterStarter/Configuration.cs create mode 100644 Tools/TestClusterStarter/Program.cs create mode 100644 Tools/TestClusterStarter/TestClusterStarter.csproj create mode 100644 docker/build-deployandrun.bat diff --git a/Tools/TestClusterStarter/ClusterTestSpec.cs b/Tools/TestClusterStarter/ClusterTestSpec.cs new file mode 100644 index 00000000..88b18c2e --- /dev/null +++ b/Tools/TestClusterStarter/ClusterTestSpec.cs @@ -0,0 +1,42 @@ +using KubernetesWorkflow; + +namespace TestClusterStarter +{ + public class ClusterTestSetup + { + public ClusterTestSetup(ClusterTestSpec[] specs) + { + Specs = specs; + } + + public ClusterTestSpec[] Specs { get; } + } + + public class ClusterTestSpec + { + public ClusterTestSpec(string name, string filter, int replication, int durationSeconds, string? codexImageOverride) + { + Name = name; + Filter = filter; + Replication = replication; + DurationSeconds = durationSeconds; + CodexImageOverride = codexImageOverride; + } + + public string Name { get; } + public string Filter { get; } + public int Replication { get; } + public int DurationSeconds { get; } + public string? CodexImageOverride { get; } + } + + public class ClusterTestDeployment + { + public ClusterTestDeployment(RunningContainer[] containers) + { + Containers = containers; + } + + public RunningContainer[] Containers { get; } + } +} diff --git a/Tools/TestClusterStarter/Configuration.cs b/Tools/TestClusterStarter/Configuration.cs new file mode 100644 index 00000000..8e30d93c --- /dev/null +++ b/Tools/TestClusterStarter/Configuration.cs @@ -0,0 +1,10 @@ +using ArgsUniform; + +namespace TestClusterStarter +{ + public class Configuration + { + [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] + public string KubeConfigFile { get; set; } = "null"; + } +} diff --git a/Tools/TestClusterStarter/Program.cs b/Tools/TestClusterStarter/Program.cs new file mode 100644 index 00000000..13dacfca --- /dev/null +++ b/Tools/TestClusterStarter/Program.cs @@ -0,0 +1,50 @@ +using ArgsUniform; +using Core; +using DeployAndRunPlugin; +using KubernetesWorkflow; +using Logging; +using Newtonsoft.Json; +using TestClusterStarter; + +public class Program +{ + private const string SpecsFile = "TestSpecs.json"; + + public static void Main(string[] args) + { + var argsUniform = new ArgsUniform(() => { }, args); + var config = argsUniform.Parse(); + + ProjectPlugin.Load(); + + if (!File.Exists(SpecsFile)) + { + File.WriteAllText(SpecsFile, JsonConvert.SerializeObject(new ClusterTestSetup(new[] + { + new ClusterTestSpec("example", "peer", 2, Convert.ToInt32(TimeSpan.FromDays(2).TotalSeconds), "imageoverride") + }))); + return; + } + var specs = JsonConvert.DeserializeObject(File.ReadAllText(SpecsFile))!; + + var kConfig = new KubernetesWorkflow.Configuration(config.KubeConfigFile, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10), "codex-continuous-test-runners"); + var entryPoint = new EntryPoint(new ConsoleLog(), kConfig, "datafolder"); + var ci = entryPoint.CreateInterface(); + + var rcs = new List(); + foreach (var spec in specs.Specs) + { + var rc = ci.DeployAndRunContinuousTests(new RunConfig( + name: spec.Name, + filter: spec.Filter, + duration: TimeSpan.FromSeconds(spec.DurationSeconds), + replications: spec.Replication, + codexImageOverride: spec.CodexImageOverride)); + + rcs.Add(rc); + } + + var deployment = new ClusterTestDeployment(rcs.ToArray()); + File.WriteAllText("clustertest-deployment.json", JsonConvert.SerializeObject(deployment, Formatting.Indented)); + } +} diff --git a/Tools/TestClusterStarter/TestClusterStarter.csproj b/Tools/TestClusterStarter/TestClusterStarter.csproj new file mode 100644 index 00000000..5522cbed --- /dev/null +++ b/Tools/TestClusterStarter/TestClusterStarter.csproj @@ -0,0 +1,21 @@ + + + + Exe + net7.0 + enable + enable + + + + + + + + + + + + + + diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 585e0026..944f83bc 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -47,6 +47,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BiblioTech", "Tools\BiblioT EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexDiscordBotPlugin", "ProjectPlugins\CodexDiscordBotPlugin\CodexDiscordBotPlugin.csproj", "{FB96A58B-F7F0-490A-9A85-72A96A018042}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestClusterStarter", "Tools\TestClusterStarter\TestClusterStarter.csproj", "{3E38A906-C2FC-43DC-8CA2-FC07C79CF3CA}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DeployAndRunPlugin", "ProjectPlugins\DeployAndRunPlugin\DeployAndRunPlugin.csproj", "{1CC5AF82-8924-4C7E-BFF1-3125D86E53FB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -125,6 +129,14 @@ Global {FB96A58B-F7F0-490A-9A85-72A96A018042}.Debug|Any CPU.Build.0 = Debug|Any CPU {FB96A58B-F7F0-490A-9A85-72A96A018042}.Release|Any CPU.ActiveCfg = Release|Any CPU {FB96A58B-F7F0-490A-9A85-72A96A018042}.Release|Any CPU.Build.0 = Release|Any CPU + {3E38A906-C2FC-43DC-8CA2-FC07C79CF3CA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3E38A906-C2FC-43DC-8CA2-FC07C79CF3CA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3E38A906-C2FC-43DC-8CA2-FC07C79CF3CA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3E38A906-C2FC-43DC-8CA2-FC07C79CF3CA}.Release|Any CPU.Build.0 = Release|Any CPU + {1CC5AF82-8924-4C7E-BFF1-3125D86E53FB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1CC5AF82-8924-4C7E-BFF1-3125D86E53FB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1CC5AF82-8924-4C7E-BFF1-3125D86E53FB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1CC5AF82-8924-4C7E-BFF1-3125D86E53FB}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -148,6 +160,8 @@ Global {3417D508-E2F4-4974-8988-BB124046D9E2} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} {078ABA6D-A04E-4F62-A44C-EA66F1B66548} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} {FB96A58B-F7F0-490A-9A85-72A96A018042} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124} + {3E38A906-C2FC-43DC-8CA2-FC07C79CF3CA} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} + {1CC5AF82-8924-4C7E-BFF1-3125D86E53FB} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C} diff --git a/docker/build-deployandrun.bat b/docker/build-deployandrun.bat new file mode 100644 index 00000000..8af14b00 --- /dev/null +++ b/docker/build-deployandrun.bat @@ -0,0 +1,2 @@ +docker build -f deployandrun.Dockerfile -t thatbenbierens/dist-tests-deployandrun:initial .. +docker push thatbenbierens/dist-tests-deployandrun:initial From b5e5570145d4a85f8373d33f5e2c80c4d67306f3 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 31 Oct 2023 11:38:54 +0100 Subject: [PATCH 04/10] Creates volume mount for kubeconfig file --- Framework/KubernetesWorkflow/ContainerRecipe.cs | 6 ++++-- .../KubernetesWorkflow/ContainerRecipeFactory.cs | 7 ++++++- Framework/KubernetesWorkflow/K8sController.cs | 14 +++++++++----- .../DeployAndRunContainerRecipe.cs | 2 ++ 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/Framework/KubernetesWorkflow/ContainerRecipe.cs b/Framework/KubernetesWorkflow/ContainerRecipe.cs index 5123b8de..a4a7f8d5 100644 --- a/Framework/KubernetesWorkflow/ContainerRecipe.cs +++ b/Framework/KubernetesWorkflow/ContainerRecipe.cs @@ -112,15 +112,17 @@ public class VolumeMount { - public VolumeMount(string volumeName, string mountPath, string resourceQuantity) + public VolumeMount(string volumeName, string mountPath, string? subPath = null, string? resourceQuantity = null) { VolumeName = volumeName; MountPath = mountPath; + SubPath = subPath; ResourceQuantity = resourceQuantity; } public string VolumeName { get; } public string MountPath { get; } - public string ResourceQuantity { get; } + public string? SubPath { get; } + public string? ResourceQuantity { get; } } } diff --git a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs index 5fa99808..290f85f9 100644 --- a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -97,12 +97,17 @@ namespace KubernetesWorkflow podAnnotations.Add(name, value); } + protected void AddVolume(string name, string mountPath, string subPath) + { + volumeMounts.Add(new VolumeMount(name, mountPath, subPath)); + } + protected void AddVolume(string mountPath, ByteSize volumeSize) { volumeMounts.Add(new VolumeMount( $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}", mountPath, - volumeSize.ToSuffixNotation())); + resourceQuantity: volumeSize.ToSuffixNotation())); } protected void Additional(object userData) diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 3557a123..75d3802c 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -441,7 +441,8 @@ namespace KubernetesWorkflow return new V1VolumeMount { Name = v.VolumeName, - MountPath = v.MountPath + MountPath = v.MountPath, + SubPath = v.SubPath, }; } @@ -457,6 +458,12 @@ namespace KubernetesWorkflow private V1Volume CreateVolume(VolumeMount v) { + var resourcesRequests = new Dictionary(); + if (v.ResourceQuantity != null) + { + resourcesRequests.Add("storage", new ResourceQuantity(v.ResourceQuantity)); + } + client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim { ApiVersion = "v1", @@ -472,10 +479,7 @@ namespace KubernetesWorkflow }, Resources = new V1ResourceRequirements { - Requests = new Dictionary - { - {"storage", new ResourceQuantity(v.ResourceQuantity) } - } + Requests = resourcesRequests } } }, K8sNamespace)); diff --git a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs index 12435a88..55605a38 100644 --- a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs +++ b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs @@ -20,6 +20,8 @@ namespace DeployAndRunPlugin AddEnvVar("DNR_NAME", setup.Name); AddEnvVar("DNR_FILTER", setup.Filter); AddEnvVar("DNR_DURATION", setup.Duration.TotalSeconds.ToString()); + + AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml"); } } From ac07327d77e1d9feefe99b553e8342d4077da082 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 31 Oct 2023 14:20:50 +0100 Subject: [PATCH 05/10] Upgrades volume support for use with deploy-and-run container --- .../KubernetesWorkflow/ContainerRecipe.cs | 6 +- .../ContainerRecipeFactory.cs | 5 +- Framework/KubernetesWorkflow/K8sController.cs | 58 +++++++++++++++---- .../DeployAndRunContainerRecipe.cs | 6 +- .../DeployAndRunPlugin/DeployAndRunPlugin.cs | 3 +- Tools/TestClusterStarter/Program.cs | 2 +- 6 files changed, 62 insertions(+), 18 deletions(-) diff --git a/Framework/KubernetesWorkflow/ContainerRecipe.cs b/Framework/KubernetesWorkflow/ContainerRecipe.cs index a4a7f8d5..4cc50999 100644 --- a/Framework/KubernetesWorkflow/ContainerRecipe.cs +++ b/Framework/KubernetesWorkflow/ContainerRecipe.cs @@ -112,17 +112,21 @@ public class VolumeMount { - public VolumeMount(string volumeName, string mountPath, string? subPath = null, string? resourceQuantity = null) + public VolumeMount(string volumeName, string mountPath, string? subPath = null, string? resourceQuantity = null, string? secret = null, string? hostPath = null) { VolumeName = volumeName; MountPath = mountPath; SubPath = subPath; ResourceQuantity = resourceQuantity; + Secret = secret; + HostPath = hostPath; } public string VolumeName { get; } public string MountPath { get; } public string? SubPath { get; } public string? ResourceQuantity { get; } + public string? Secret { get; } + public string? HostPath { get; } } } diff --git a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs index 290f85f9..d5273dd9 100644 --- a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -97,9 +97,10 @@ namespace KubernetesWorkflow podAnnotations.Add(name, value); } - protected void AddVolume(string name, string mountPath, string subPath) + protected void AddVolume(string name, string mountPath, string? subPath = null, string? secret = null, string? hostPath = null) { - volumeMounts.Add(new VolumeMount(name, mountPath, subPath)); + var size = 10.MB().ToSuffixNotation(); + volumeMounts.Add(new VolumeMount(name, mountPath, subPath, size, secret, hostPath)); } protected void AddVolume(string mountPath, ByteSize volumeSize) diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 75d3802c..f18afdb2 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -458,32 +458,45 @@ namespace KubernetesWorkflow private V1Volume CreateVolume(VolumeMount v) { - var resourcesRequests = new Dictionary(); - if (v.ResourceQuantity != null) - { - resourcesRequests.Add("storage", new ResourceQuantity(v.ResourceQuantity)); - } - client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim { ApiVersion = "v1", Metadata = new V1ObjectMeta { - Name = v.VolumeName + Name = v.VolumeName, }, Spec = new V1PersistentVolumeClaimSpec { + AccessModes = new List { "ReadWriteOnce" }, - Resources = new V1ResourceRequirements - { - Requests = resourcesRequests - } - } + Resources = CreateVolumeResourceRequirements(v), + }, }, K8sNamespace)); + if (!string.IsNullOrEmpty(v.HostPath)) + { + return new V1Volume + { + Name = v.VolumeName, + HostPath = new V1HostPathVolumeSource + { + Path = v.HostPath + } + }; + } + + if (!string.IsNullOrEmpty(v.Secret)) + { + return new V1Volume + { + Name = v.VolumeName, + Secret = CreateVolumeSecret(v) + }; + } + return new V1Volume { Name = v.VolumeName, @@ -494,6 +507,27 @@ namespace KubernetesWorkflow }; } + private V1SecretVolumeSource CreateVolumeSecret(VolumeMount v) + { + if (string.IsNullOrWhiteSpace(v.Secret)) return null!; + return new V1SecretVolumeSource + { + SecretName = v.Secret + }; + } + + private V1ResourceRequirements CreateVolumeResourceRequirements(VolumeMount v) + { + if (v.ResourceQuantity == null) return null!; + return new V1ResourceRequirements + { + Requests = new Dictionary() + { + {"storage", new ResourceQuantity(v.ResourceQuantity) } + } + }; + } + private List CreateEnv(ContainerRecipe recipe) { return recipe.EnvVars.Select(CreateEnvVar).ToList(); diff --git a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs index 55605a38..96922259 100644 --- a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs +++ b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs @@ -21,7 +21,11 @@ namespace DeployAndRunPlugin AddEnvVar("DNR_FILTER", setup.Filter); AddEnvVar("DNR_DURATION", setup.Duration.TotalSeconds.ToString()); - AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml"); + AddEnvVar("KUBECONFIG", "/opt/kubeconfig.yaml"); + AddEnvVar("LOGPATH", "/var/log/codex-continuous-tests"); + + AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml", secret: "codex-dist-tests-app-kubeconfig"); + AddVolume(name: "logs", mountPath: "/var/log/codex-continuous-tests", hostPath: "/var/log/codex-continuous-tests"); } } diff --git a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs index 14405443..82d80e07 100644 --- a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs +++ b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs @@ -28,7 +28,8 @@ namespace DeployAndRunPlugin startupConfig.NameOverride = "dnr-" + config.Name; startupConfig.Add(config); - var containers = workflow.Start(1, new DeployAndRunContainerRecipe(), startupConfig); + var location = workflow.GetAvailableLocations().Get("fixed-s-4vcpu-16gb-amd-yz8rd"); + var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig); return containers.Containers.Single(); } } diff --git a/Tools/TestClusterStarter/Program.cs b/Tools/TestClusterStarter/Program.cs index 13dacfca..28306610 100644 --- a/Tools/TestClusterStarter/Program.cs +++ b/Tools/TestClusterStarter/Program.cs @@ -27,7 +27,7 @@ public class Program } var specs = JsonConvert.DeserializeObject(File.ReadAllText(SpecsFile))!; - var kConfig = new KubernetesWorkflow.Configuration(config.KubeConfigFile, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10), "codex-continuous-test-runners"); + var kConfig = new KubernetesWorkflow.Configuration(config.KubeConfigFile, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10), kubernetesNamespace: "default"); var entryPoint = new EntryPoint(new ConsoleLog(), kConfig, "datafolder"); var ci = entryPoint.CreateInterface(); From a6f7bc2393ee58f2b6e6575f2cfa1a62020b7614 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 31 Oct 2023 14:48:16 +0100 Subject: [PATCH 06/10] Removes knownPods class. --- Framework/KubernetesWorkflow/K8sController.cs | 39 ++++++++++--------- Framework/KubernetesWorkflow/KnownK8sPods.cs | 17 -------- .../KubernetesWorkflow/StartupWorkflow.cs | 8 ++-- .../KubernetesWorkflow/WorkflowCreator.cs | 3 +- 4 files changed, 25 insertions(+), 42 deletions(-) delete mode 100644 Framework/KubernetesWorkflow/KnownK8sPods.cs diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index f18afdb2..80548922 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -9,15 +9,14 @@ namespace KubernetesWorkflow { private readonly ILog log; private readonly K8sCluster cluster; - private readonly KnownK8sPods knownPods; private readonly WorkflowNumberSource workflowNumberSource; private readonly K8sClient client; + private const string podLabelKey = "pod-uuid"; - public K8sController(ILog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string k8sNamespace) + public K8sController(ILog log, K8sCluster cluster, WorkflowNumberSource workflowNumberSource, string k8sNamespace) { this.log = log; this.cluster = cluster; - this.knownPods = knownPods; this.workflowNumberSource = workflowNumberSource; client = new K8sClient(cluster.GetK8sClientConfig()); @@ -34,9 +33,13 @@ namespace KubernetesWorkflow log.Debug(); EnsureNamespace(); - var deploymentName = CreateDeployment(containerRecipes, location); + var podLabel = K8sNameUtils.Format(Guid.NewGuid().ToString()); + var deploymentName = CreateDeployment(containerRecipes, location, podLabel); var (serviceName, servicePortsMap) = CreateService(containerRecipes); - var podInfo = FetchNewPod(); + + var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)); + var pod = pods.Items.Single(p => p.Labels().Any(l => l.Key == podLabelKey && l.Value == podLabel)); + var podInfo = CreatePodInfo(pod); return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray()); } @@ -299,7 +302,7 @@ namespace KubernetesWorkflow #region Deployment management - private string CreateDeployment(ContainerRecipe[] containerRecipes, ILocation location) + private string CreateDeployment(ContainerRecipe[] containerRecipes, ILocation location, string podLabel) { var deploymentSpec = new V1Deployment { @@ -316,7 +319,7 @@ namespace KubernetesWorkflow { Metadata = new V1ObjectMeta { - Labels = GetSelector(containerRecipes), + Labels = GetSelector(containerRecipes, podLabel), Annotations = GetAnnotations(containerRecipes) }, Spec = new V1PodSpec @@ -363,6 +366,13 @@ namespace KubernetesWorkflow return containerRecipes.First().PodLabels.GetLabels(); } + private IDictionary GetSelector(ContainerRecipe[] containerRecipes, string podLabel) + { + var labels = containerRecipes.First().PodLabels.Clone(); + labels.Add(podLabelKey, podLabel); + return labels.GetLabels(); + } + private IDictionary GetRunnerNamespaceSelector() { return new Dictionary { { "kubernetes.io/metadata.name", "default" } }; @@ -751,22 +761,15 @@ namespace KubernetesWorkflow return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sNamespace, container); } - private PodInfo FetchNewPod() + private PodInfo CreatePodInfo(V1Pod pod) { - var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)).Items; - - var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); - if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure."); - - var newPod = newPods.Single(); - var name = newPod.Name(); - var ip = newPod.Status.PodIP; - var k8sNodeName = newPod.Spec.NodeName; + var name = pod.Name(); + var ip = pod.Status.PodIP; + var k8sNodeName = pod.Spec.NodeName; if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure."); if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure."); - knownPods.Add(name); return new PodInfo(name, ip, k8sNodeName); } } diff --git a/Framework/KubernetesWorkflow/KnownK8sPods.cs b/Framework/KubernetesWorkflow/KnownK8sPods.cs deleted file mode 100644 index 6d80eb66..00000000 --- a/Framework/KubernetesWorkflow/KnownK8sPods.cs +++ /dev/null @@ -1,17 +0,0 @@ -namespace KubernetesWorkflow -{ - public class KnownK8sPods - { - private readonly List knownActivePodNames = new List(); - - public bool Contains(string name) - { - return knownActivePodNames.Contains(name); - } - - public void Add(string name) - { - knownActivePodNames.Add(name); - } - } -} diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs index 1cd5eb1a..b34cb751 100644 --- a/Framework/KubernetesWorkflow/StartupWorkflow.cs +++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs @@ -22,17 +22,15 @@ namespace KubernetesWorkflow private readonly ILog log; private readonly WorkflowNumberSource numberSource; private readonly K8sCluster cluster; - private readonly KnownK8sPods knownK8SPods; private readonly string k8sNamespace; private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory(); private readonly LocationProvider locationProvider; - internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string k8sNamespace) + internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, string k8sNamespace) { this.log = log; this.numberSource = numberSource; this.cluster = cluster; - this.knownK8SPods = knownK8SPods; this.k8sNamespace = k8sNamespace; locationProvider = new LocationProvider(log, K8s); @@ -196,7 +194,7 @@ namespace KubernetesWorkflow { try { - var controller = new K8sController(log, cluster, knownK8SPods, numberSource, k8sNamespace); + var controller = new K8sController(log, cluster, numberSource, k8sNamespace); action(controller); controller.Dispose(); } @@ -211,7 +209,7 @@ namespace KubernetesWorkflow { try { - var controller = new K8sController(log, cluster, knownK8SPods, numberSource, k8sNamespace); + var controller = new K8sController(log, cluster, numberSource, k8sNamespace); var result = action(controller); controller.Dispose(); return result; diff --git a/Framework/KubernetesWorkflow/WorkflowCreator.cs b/Framework/KubernetesWorkflow/WorkflowCreator.cs index 213db929..d70b0bcf 100644 --- a/Framework/KubernetesWorkflow/WorkflowCreator.cs +++ b/Framework/KubernetesWorkflow/WorkflowCreator.cs @@ -7,7 +7,6 @@ namespace KubernetesWorkflow { private readonly NumberSource numberSource = new NumberSource(0); private readonly NumberSource containerNumberSource = new NumberSource(0); - private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly K8sCluster cluster; private readonly ILog log; private readonly Configuration configuration; @@ -26,7 +25,7 @@ namespace KubernetesWorkflow var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), containerNumberSource); - return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, GetNamespace(namespaceOverride)); + return new StartupWorkflow(log, workflowNumberSource, cluster, GetNamespace(namespaceOverride)); } private string GetNamespace(string? namespaceOverride) From fcb5a527a95696b1a14aba0105d414f7358c4a4b Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 31 Oct 2023 15:04:59 +0100 Subject: [PATCH 07/10] Fixes pod-readback labeling issue. --- Framework/KubernetesWorkflow/K8sController.cs | 59 +++++++++++++------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 80548922..d150198c 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -37,13 +37,26 @@ namespace KubernetesWorkflow var deploymentName = CreateDeployment(containerRecipes, location, podLabel); var (serviceName, servicePortsMap) = CreateService(containerRecipes); - var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)); - var pod = pods.Items.Single(p => p.Labels().Any(l => l.Key == podLabelKey && l.Value == podLabel)); + var pod = FindPodByLabel(podLabel); var podInfo = CreatePodInfo(pod); return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray()); } + private V1Pod FindPodByLabel(string podLabel) + { + var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)); + foreach (var pod in pods.Items) + { + var label = pod.GetLabel(podLabelKey); + if (label == podLabel) + { + return pod; + } + } + throw new Exception("Unable to find pod by label."); + } + public void Stop(RunningPod pod) { log.Debug(); @@ -468,23 +481,7 @@ namespace KubernetesWorkflow private V1Volume CreateVolume(VolumeMount v) { - client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = v.VolumeName, - }, - Spec = new V1PersistentVolumeClaimSpec - { - - AccessModes = new List - { - "ReadWriteOnce" - }, - Resources = CreateVolumeResourceRequirements(v), - }, - }, K8sNamespace)); + CreatePersistentVolumeClaimIfNeeded(v); if (!string.IsNullOrEmpty(v.HostPath)) { @@ -517,6 +514,30 @@ namespace KubernetesWorkflow }; } + private void CreatePersistentVolumeClaimIfNeeded(VolumeMount v) + { + var pvcs = client.Run(c => c.ListNamespacedPersistentVolumeClaim(K8sNamespace)); + if (pvcs != null && pvcs.Items.Any(i => i.Name() != v.VolumeName)) return; + + client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = v.VolumeName, + }, + Spec = new V1PersistentVolumeClaimSpec + { + + AccessModes = new List + { + "ReadWriteOnce" + }, + Resources = CreateVolumeResourceRequirements(v), + }, + }, K8sNamespace)); + } + private V1SecretVolumeSource CreateVolumeSecret(VolumeMount v) { if (string.IsNullOrWhiteSpace(v.Secret)) return null!; From 49300273e0471680dd50235050272b482f75cf3b Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 31 Oct 2023 15:33:00 +0100 Subject: [PATCH 08/10] Cleanup continuous tests nicely even if no tests were selected for running. --- .../ContinuousTestRunner.cs | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/Tests/CodexContinuousTests/ContinuousTestRunner.cs b/Tests/CodexContinuousTests/ContinuousTestRunner.cs index 2566d1d6..8461f7f9 100644 --- a/Tests/CodexContinuousTests/ContinuousTestRunner.cs +++ b/Tests/CodexContinuousTests/ContinuousTestRunner.cs @@ -53,22 +53,25 @@ namespace ContinuousTests if (!filteredTests.Any()) { overviewLog.Log("No tests selected."); - return; + Cancellation.Cts.Cancel(); } - var testLoops = filteredTests.Select(t => new TestLoop(entryPointFactory, taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray(); - - foreach (var testLoop in testLoops) + else { - if (cancelToken.IsCancellationRequested) break; + var testLoops = filteredTests.Select(t => new TestLoop(entryPointFactory, taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray(); - overviewLog.Log("Launching test-loop for " + testLoop.Name); - testLoop.Begin(); - Thread.Sleep(TimeSpan.FromSeconds(5)); + foreach (var testLoop in testLoops) + { + if (cancelToken.IsCancellationRequested) break; + + overviewLog.Log("Launching test-loop for " + testLoop.Name); + testLoop.Begin(); + Thread.Sleep(TimeSpan.FromSeconds(5)); + } + + overviewLog.Log("Finished launching test-loops."); + WaitUntilFinished(overviewLog, statusLog, startTime, testLoops); + overviewLog.Log("Stopping all test-loops..."); } - - overviewLog.Log("Finished launching test-loops."); - WaitUntilFinished(overviewLog, statusLog, startTime, testLoops); - overviewLog.Log("Stopping all test-loops..."); taskFactory.WaitAll(); overviewLog.Log("All tasks cancelled."); From 5241144e995f704d449bfd0665cecf891bce8c84 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 1 Nov 2023 10:27:31 +0100 Subject: [PATCH 09/10] Locks up deploy-and-run script after tests are started. --- Tests/CodexContinuousTests/deploy-and-run.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tests/CodexContinuousTests/deploy-and-run.sh b/Tests/CodexContinuousTests/deploy-and-run.sh index 5dd7a5ea..3d778adf 100644 --- a/Tests/CodexContinuousTests/deploy-and-run.sh +++ b/Tests/CodexContinuousTests/deploy-and-run.sh @@ -50,3 +50,6 @@ do sleep 30 done + +echo "Done! Sleeping indefinitely..." +while true; do sleep 1d; done From bc51fc2e309880385714d925aaa9f10391f0ab70 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 2 Nov 2023 11:32:24 +0100 Subject: [PATCH 10/10] Fixes faulty persistent volume claim creation --- Framework/KubernetesWorkflow/K8sController.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index d150198c..b9b4386f 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -517,7 +517,7 @@ namespace KubernetesWorkflow private void CreatePersistentVolumeClaimIfNeeded(VolumeMount v) { var pvcs = client.Run(c => c.ListNamespacedPersistentVolumeClaim(K8sNamespace)); - if (pvcs != null && pvcs.Items.Any(i => i.Name() != v.VolumeName)) return; + if (pvcs != null && pvcs.Items.Any(i => i.Name() == v.VolumeName)) return; client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim { @@ -528,11 +528,10 @@ namespace KubernetesWorkflow }, Spec = new V1PersistentVolumeClaimSpec { - AccessModes = new List - { - "ReadWriteOnce" - }, + { + "ReadWriteOnce" + }, Resources = CreateVolumeResourceRequirements(v), }, }, K8sNamespace));