From ac07327d77e1d9feefe99b553e8342d4077da082 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 31 Oct 2023 14:20:50 +0100 Subject: [PATCH] Upgrades volume support for use with deploy-and-run container --- .../KubernetesWorkflow/ContainerRecipe.cs | 6 +- .../ContainerRecipeFactory.cs | 5 +- Framework/KubernetesWorkflow/K8sController.cs | 58 +++++++++++++++---- .../DeployAndRunContainerRecipe.cs | 6 +- .../DeployAndRunPlugin/DeployAndRunPlugin.cs | 3 +- Tools/TestClusterStarter/Program.cs | 2 +- 6 files changed, 62 insertions(+), 18 deletions(-) diff --git a/Framework/KubernetesWorkflow/ContainerRecipe.cs b/Framework/KubernetesWorkflow/ContainerRecipe.cs index a4a7f8d..4cc5099 100644 --- a/Framework/KubernetesWorkflow/ContainerRecipe.cs +++ b/Framework/KubernetesWorkflow/ContainerRecipe.cs @@ -112,17 +112,21 @@ public class VolumeMount { - public VolumeMount(string volumeName, string mountPath, string? subPath = null, string? resourceQuantity = null) + public VolumeMount(string volumeName, string mountPath, string? subPath = null, string? resourceQuantity = null, string? secret = null, string? hostPath = null) { VolumeName = volumeName; MountPath = mountPath; SubPath = subPath; ResourceQuantity = resourceQuantity; + Secret = secret; + HostPath = hostPath; } public string VolumeName { get; } public string MountPath { get; } public string? SubPath { get; } public string? ResourceQuantity { get; } + public string? Secret { get; } + public string? HostPath { get; } } } diff --git a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs index 290f85f..d5273dd 100644 --- a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -97,9 +97,10 @@ namespace KubernetesWorkflow podAnnotations.Add(name, value); } - protected void AddVolume(string name, string mountPath, string subPath) + protected void AddVolume(string name, string mountPath, string? subPath = null, string? secret = null, string? hostPath = null) { - volumeMounts.Add(new VolumeMount(name, mountPath, subPath)); + var size = 10.MB().ToSuffixNotation(); + volumeMounts.Add(new VolumeMount(name, mountPath, subPath, size, secret, hostPath)); } protected void AddVolume(string mountPath, ByteSize volumeSize) diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 75d3802..f18afdb 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -458,32 +458,45 @@ namespace KubernetesWorkflow private V1Volume CreateVolume(VolumeMount v) { - var resourcesRequests = new Dictionary(); - if (v.ResourceQuantity != null) - { - resourcesRequests.Add("storage", new ResourceQuantity(v.ResourceQuantity)); - } - client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim { ApiVersion = "v1", Metadata = new V1ObjectMeta { - Name = v.VolumeName + Name = v.VolumeName, }, Spec = new V1PersistentVolumeClaimSpec { + AccessModes = new List { "ReadWriteOnce" }, - Resources = new V1ResourceRequirements - { - Requests = resourcesRequests - } - } + Resources = CreateVolumeResourceRequirements(v), + }, }, K8sNamespace)); + if (!string.IsNullOrEmpty(v.HostPath)) + { + return new V1Volume + { + Name = v.VolumeName, + HostPath = new V1HostPathVolumeSource + { + Path = v.HostPath + } + }; + } + + if (!string.IsNullOrEmpty(v.Secret)) + { + return new V1Volume + { + Name = v.VolumeName, + Secret = CreateVolumeSecret(v) + }; + } + return new V1Volume { Name = v.VolumeName, @@ -494,6 +507,27 @@ namespace KubernetesWorkflow }; } + private V1SecretVolumeSource CreateVolumeSecret(VolumeMount v) + { + if (string.IsNullOrWhiteSpace(v.Secret)) return null!; + return new V1SecretVolumeSource + { + SecretName = v.Secret + }; + } + + private V1ResourceRequirements CreateVolumeResourceRequirements(VolumeMount v) + { + if (v.ResourceQuantity == null) return null!; + return new V1ResourceRequirements + { + Requests = new Dictionary() + { + {"storage", new ResourceQuantity(v.ResourceQuantity) } + } + }; + } + private List CreateEnv(ContainerRecipe recipe) { return recipe.EnvVars.Select(CreateEnvVar).ToList(); diff --git a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs index 55605a3..9692225 100644 --- a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs +++ b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs @@ -21,7 +21,11 @@ namespace DeployAndRunPlugin AddEnvVar("DNR_FILTER", setup.Filter); AddEnvVar("DNR_DURATION", setup.Duration.TotalSeconds.ToString()); - AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml"); + AddEnvVar("KUBECONFIG", "/opt/kubeconfig.yaml"); + AddEnvVar("LOGPATH", "/var/log/codex-continuous-tests"); + + AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml", secret: "codex-dist-tests-app-kubeconfig"); + AddVolume(name: "logs", mountPath: "/var/log/codex-continuous-tests", hostPath: "/var/log/codex-continuous-tests"); } } diff --git a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs index 1440544..82d80e0 100644 --- a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs +++ b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs @@ -28,7 +28,8 @@ namespace DeployAndRunPlugin startupConfig.NameOverride = "dnr-" + config.Name; startupConfig.Add(config); - var containers = workflow.Start(1, new DeployAndRunContainerRecipe(), startupConfig); + var location = workflow.GetAvailableLocations().Get("fixed-s-4vcpu-16gb-amd-yz8rd"); + var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig); return containers.Containers.Single(); } } diff --git a/Tools/TestClusterStarter/Program.cs b/Tools/TestClusterStarter/Program.cs index 13dacfc..2830661 100644 --- a/Tools/TestClusterStarter/Program.cs +++ b/Tools/TestClusterStarter/Program.cs @@ -27,7 +27,7 @@ public class Program } var specs = JsonConvert.DeserializeObject(File.ReadAllText(SpecsFile))!; - var kConfig = new KubernetesWorkflow.Configuration(config.KubeConfigFile, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10), "codex-continuous-test-runners"); + var kConfig = new KubernetesWorkflow.Configuration(config.KubeConfigFile, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(10), kubernetesNamespace: "default"); var entryPoint = new EntryPoint(new ConsoleLog(), kConfig, "datafolder"); var ci = entryPoint.CreateInterface();