Creates persistent-volume-claims for codex nodes

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -31,7 +31,7 @@ namespace DistTestCore.Codex
 
             var dataDir = $"datadir{ContainerNumber}";
             AddEnvVar("CODEX_DATA_DIR", dataDir);
-            AddVolume(dataDir);
+            AddVolume($"codex/{dataDir}", GetVolumeCapacity(config));
 
             AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
             AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
@@ -94,6 +94,13 @@ namespace DistTestCore.Codex
             }
         }
 
+        private long GetVolumeCapacity(CodexStartupConfig config)
+        {
+            if (config.StorageQuota != null) return config.StorageQuota.SizeInBytes;
+            // Default Codex quota: 8 Gb, using 9 to be safe.
+            return 9.GB().SizeInBytes;
+        }
+
         private int GetAccountIndex(MarketplaceInitialConfig marketplaceConfig)
         {
             if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value;

KubernetesWorkflow/ContainerRecipe.cs

@@ -67,13 +67,15 @@
 
     public class VolumeMount
     {
-        public VolumeMount(string volumeName, string mountPath)
+        public VolumeMount(string volumeName, string mountPath, string resourceQuantity)
         {
             VolumeName = volumeName;
             MountPath = mountPath;
+            ResourceQuantity = resourceQuantity;
         }
 
         public string VolumeName { get; }
         public string MountPath { get; }
+        public string ResourceQuantity { get; }
     }
 }

KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -97,16 +97,38 @@
             podAnnotations.Add(name, value);
         }
 
-        protected void AddVolume(string mountPath)
+        protected void AddVolume(string mountPath, long capacityBytes)
         {
             volumeMounts.Add(new VolumeMount(
                 $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}",
-                mountPath));
+                mountPath,
+                FormatBytesQuantity(capacityBytes)));
         }
 
         protected void Additional(object userData)
         {
             additionals.Add(userData);
         }
+
+        private static string FormatBytesQuantity(long capacityBytes)
+        {
+            var map = new Dictionary<long, string>
+            {
+                { (1024*1024*1024), "Gi" },
+                { (1024*1024), "Mi" },
+                { (1024), "Ki" },
+            };
+
+            foreach (var pair in map)
+            {
+                if (capacityBytes > pair.Key)
+                {
+                    var v = (capacityBytes / pair.Key) + 1;
+                    return $"{v}{pair.Value}";
+                }
+            }
+
+            return $"{capacityBytes * 2}Ki";
+        }
     }
 }

KubernetesWorkflow/K8sController.cs

@@ -1,6 +1,9 @@
 using k8s;
 using k8s.Models;
 using Logging;
+using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources;
+using System.Collections.Generic;
+using System.Xml.Linq;
 using Utils;
 
 namespace KubernetesWorkflow
@@ -345,7 +348,8 @@ namespace KubernetesWorkflow
                         Spec = new V1PodSpec
                         {
                             NodeSelector = CreateNodeSelector(location),
-                            Containers = CreateDeploymentContainers(containerRecipes)
+                            Containers = CreateDeploymentContainers(containerRecipes),
+                            Volumes = CreateVolumes(containerRecipes)
                         }
                     }
                 }
@@ -407,7 +411,7 @@ namespace KubernetesWorkflow
 
         private List<V1Container> CreateDeploymentContainers(ContainerRecipe[] containerRecipes)
         {
-            return containerRecipes.Select(r => CreateDeploymentContainer(r)).ToList();
+            return containerRecipes.Select(CreateDeploymentContainer).ToList();
         }
 
         private V1Container CreateDeploymentContainer(ContainerRecipe recipe)
@@ -418,7 +422,67 @@ namespace KubernetesWorkflow
                 Image = recipe.Image,
                 ImagePullPolicy = "Always",
                 Ports = CreateContainerPorts(recipe),
-                Env = CreateEnv(recipe)
+                Env = CreateEnv(recipe),
+                VolumeMounts = CreateContainerVolumeMounts(recipe)
+            };
+        }
+
+        private List<V1VolumeMount> CreateContainerVolumeMounts(ContainerRecipe recipe)
+        {
+            return recipe.Volumes.Select(CreateContainerVolumeMount).ToList();
+        }
+
+        private V1VolumeMount CreateContainerVolumeMount(VolumeMount v)
+        {
+            return new V1VolumeMount
+            {
+                Name = v.VolumeName,
+                MountPath = v.MountPath
+            };
+        }
+
+        private List<V1Volume> CreateVolumes(ContainerRecipe[] containerRecipes)
+        {
+            return containerRecipes.Where(c => c.Volumes.Any()).SelectMany(CreateVolumes).ToList();
+        }
+
+        private List<V1Volume> CreateVolumes(ContainerRecipe recipe)
+        {
+            return recipe.Volumes.Select(CreateVolume).ToList();
+        }
+
+        private V1Volume CreateVolume(VolumeMount v)
+        {
+            client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim
+            {
+                ApiVersion = "v1",
+                Metadata = new V1ObjectMeta
+                {
+                    Name = v.VolumeName
+                },
+                Spec = new V1PersistentVolumeClaimSpec
+                {
+                    AccessModes = new List<string>
+                    {
+                        "ReadWriteMany"
+                    },
+                    Resources = new V1ResourceRequirements
+                    {
+                        Requests = new Dictionary<string, ResourceQuantity>
+                        {
+                            {"storage", new ResourceQuantity(v.ResourceQuantity) }
+                        }
+                    }
+                }
+            }, K8sTestNamespace));
+
+            return new V1Volume
+            {
+                Name = v.VolumeName,
+                PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource
+                {
+                    ClaimName = v.VolumeName
+                }
             };
         }