From 999ed6cfc8fc5c8ad77b0eee1428a0f12bb2ea2d Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 7 Sep 2023 10:37:52 +0200 Subject: [PATCH] Creates persistent-volume-claims for codex nodes --- DistTestCore/Codex/CodexContainerRecipe.cs | 9 ++- KubernetesWorkflow/ContainerRecipe.cs | 4 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 26 +++++++- KubernetesWorkflow/K8sController.cs | 70 +++++++++++++++++++- 4 files changed, 102 insertions(+), 7 deletions(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 1b03a50..b38b3e7 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -31,7 +31,7 @@ namespace DistTestCore.Codex var dataDir = $"datadir{ContainerNumber}"; AddEnvVar("CODEX_DATA_DIR", dataDir); - AddVolume(dataDir); + AddVolume($"codex/{dataDir}", GetVolumeCapacity(config)); AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag); AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); @@ -94,6 +94,13 @@ namespace DistTestCore.Codex } } + private long GetVolumeCapacity(CodexStartupConfig config) + { + if (config.StorageQuota != null) return config.StorageQuota.SizeInBytes; + // Default Codex quota: 8 Gb, using 9 to be safe. + return 9.GB().SizeInBytes; + } + private int GetAccountIndex(MarketplaceInitialConfig marketplaceConfig) { if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value; diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index f58c3fd..ed16b49 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -67,13 +67,15 @@ public class VolumeMount { - public VolumeMount(string volumeName, string mountPath) + public VolumeMount(string volumeName, string mountPath, string resourceQuantity) { VolumeName = volumeName; MountPath = mountPath; + ResourceQuantity = resourceQuantity; } public string VolumeName { get; } public string MountPath { get; } + public string ResourceQuantity { get; } } } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index f10d062..d5c75a4 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -97,16 +97,38 @@ podAnnotations.Add(name, value); } - protected void AddVolume(string mountPath) + protected void AddVolume(string mountPath, long capacityBytes) { volumeMounts.Add(new VolumeMount( $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}", - mountPath)); + mountPath, + FormatBytesQuantity(capacityBytes))); } protected void Additional(object userData) { additionals.Add(userData); } + + private static string FormatBytesQuantity(long capacityBytes) + { + var map = new Dictionary + { + { (1024*1024*1024), "Gi" }, + { (1024*1024), "Mi" }, + { (1024), "Ki" }, + }; + + foreach (var pair in map) + { + if (capacityBytes > pair.Key) + { + var v = (capacityBytes / pair.Key) + 1; + return $"{v}{pair.Value}"; + } + } + + return $"{capacityBytes * 2}Ki"; + } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index d06070d..f691f7c 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -1,6 +1,9 @@ using k8s; using k8s.Models; using Logging; +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources; +using System.Collections.Generic; +using System.Xml.Linq; using Utils; namespace KubernetesWorkflow @@ -345,7 +348,8 @@ namespace KubernetesWorkflow Spec = new V1PodSpec { NodeSelector = CreateNodeSelector(location), - Containers = CreateDeploymentContainers(containerRecipes) + Containers = CreateDeploymentContainers(containerRecipes), + Volumes = CreateVolumes(containerRecipes) } } } @@ -407,7 +411,7 @@ namespace KubernetesWorkflow private List CreateDeploymentContainers(ContainerRecipe[] containerRecipes) { - return containerRecipes.Select(r => CreateDeploymentContainer(r)).ToList(); + return containerRecipes.Select(CreateDeploymentContainer).ToList(); } private V1Container CreateDeploymentContainer(ContainerRecipe recipe) @@ -418,7 +422,67 @@ namespace KubernetesWorkflow Image = recipe.Image, ImagePullPolicy = "Always", Ports = CreateContainerPorts(recipe), - Env = CreateEnv(recipe) + Env = CreateEnv(recipe), + VolumeMounts = CreateContainerVolumeMounts(recipe) + }; + } + + private List CreateContainerVolumeMounts(ContainerRecipe recipe) + { + return recipe.Volumes.Select(CreateContainerVolumeMount).ToList(); + } + + private V1VolumeMount CreateContainerVolumeMount(VolumeMount v) + { + return new V1VolumeMount + { + Name = v.VolumeName, + MountPath = v.MountPath + }; + } + + private List CreateVolumes(ContainerRecipe[] containerRecipes) + { + return containerRecipes.Where(c => c.Volumes.Any()).SelectMany(CreateVolumes).ToList(); + } + + private List CreateVolumes(ContainerRecipe recipe) + { + return recipe.Volumes.Select(CreateVolume).ToList(); + } + + private V1Volume CreateVolume(VolumeMount v) + { + client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = v.VolumeName + }, + Spec = new V1PersistentVolumeClaimSpec + { + AccessModes = new List + { + "ReadWriteMany" + }, + Resources = new V1ResourceRequirements + { + Requests = new Dictionary + { + {"storage", new ResourceQuantity(v.ResourceQuantity) } + } + } + } + }, K8sTestNamespace)); + + return new V1Volume + { + Name = v.VolumeName, + PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource + { + ClaimName = v.VolumeName + } }; }