Merge branch 'feature/automatic-volumes'

CodexNetDeployer/CodexNodeStarter.cs

@@ -2,6 +2,7 @@
 using DistTestCore.Codex;
 using DistTestCore.Marketplace;
 using KubernetesWorkflow;
+using Utils;
 
 namespace CodexNetDeployer
 {

CodexNetDeployer/Deployer.cs

@@ -2,6 +2,7 @@
 using DistTestCore.Codex;
 using KubernetesWorkflow;
 using Logging;
+using Utils;
 
 namespace CodexNetDeployer
 {

ContinuousTests/Tests/HoldMyBeerTest.cs

@@ -1,5 +1,6 @@
 using DistTestCore;
 using NUnit.Framework;
+using Utils;
 
 namespace ContinuousTests.Tests
 {

ContinuousTests/Tests/TwoClientTest.cs

@@ -1,5 +1,6 @@
 using DistTestCore;
 using NUnit.Framework;
+using Utils;
 
 namespace ContinuousTests.Tests
 {

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -1,5 +1,6 @@
 using DistTestCore.Marketplace;
 using KubernetesWorkflow;
+using Utils;
 
 namespace DistTestCore.Codex
 {
@@ -16,10 +17,13 @@ namespace DistTestCore.Codex
 
         public override string AppName => "codex";
         public override string Image { get; }
-
+        
         public CodexContainerRecipe()
         {
             Image = GetDockerImage();
+
+            Resources.Requests = new ContainerResourceSet(milliCPUs: 1000, memory: 6.GB());
+            Resources.Limits = new ContainerResourceSet(milliCPUs: 4000, memory: 12.GB());
         }
 
         protected override void InitializeRecipe(StartupConfig startupConfig)
@@ -29,7 +33,10 @@ namespace DistTestCore.Codex
             AddExposedPortAndVar("CODEX_API_PORT");
             AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
 
-            AddEnvVar("CODEX_DATA_DIR", $"datadir{ContainerNumber}");
+            var dataDir = $"datadir{ContainerNumber}";
+            AddEnvVar("CODEX_DATA_DIR", dataDir);
+            AddVolume($"codex/{dataDir}", GetVolumeCapacity(config));
+
             AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
             AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
 
@@ -91,6 +98,13 @@ namespace DistTestCore.Codex
             }
         }
 
+        private ByteSize GetVolumeCapacity(CodexStartupConfig config)
+        {
+            if (config.StorageQuota != null) return config.StorageQuota;
+            // Default Codex quota: 8 Gb, using +20% to be safe.
+            return 8.GB().Multiply(1.2);
+        }
+
         private int GetAccountIndex(MarketplaceInitialConfig marketplaceConfig)
         {
             if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value;

DistTestCore/Codex/CodexStartupConfig.cs

@@ -1,6 +1,7 @@
 using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
 using KubernetesWorkflow;
+using Utils;
 
 namespace DistTestCore.Codex
 {

DistTestCore/CodexSetup.cs

@@ -1,6 +1,7 @@
 using DistTestCore.Codex;
 using DistTestCore.Marketplace;
 using KubernetesWorkflow;
+using Utils;
 
 namespace DistTestCore
 {

DistTestCore/DistTest.cs

@@ -7,6 +7,7 @@ using KubernetesWorkflow;
 using Logging;
 using NUnit.Framework;
 using System.Reflection;
+using Utils;
 
 namespace DistTestCore
 {

DistTestCore/GrafanaStarter.cs

@@ -3,6 +3,7 @@ using IdentityModel.Client;
 using KubernetesWorkflow;
 using Newtonsoft.Json;
 using System.Reflection;
+using Utils;
 
 namespace DistTestCore
 {

DistTestCore/Helpers/FullConnectivityHelper.cs

@@ -1,7 +1,6 @@
 using DistTestCore.Codex;
 using Logging;
 using NUnit.Framework;
-using Utils;
 
 namespace DistTestCore.Helpers
 {
@@ -35,10 +34,9 @@ namespace DistTestCore.Helpers
             var entries = CreateEntries(nodes);
             var pairs = CreatePairs(entries);
 
-            RetryWhilePairs(pairs, () =>
-            {
-                CheckAndRemoveSuccessful(pairs);
-            });
+            // Each pair gets two chances.
+            CheckAndRemoveSuccessful(pairs);
+            CheckAndRemoveSuccessful(pairs);
 
             if (pairs.Any())
             {
@@ -54,35 +52,19 @@ namespace DistTestCore.Helpers
             }
         }
 
-        private static void RetryWhilePairs(List<Pair> pairs, Action action)
-        {
-            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2);
-            while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
-            {
-                action();
-
-                Time.Sleep(TimeSpan.FromSeconds(2));
-            }
-        }
-
         private void CheckAndRemoveSuccessful(List<Pair> pairs)
         {
-            // For large sets, don't try and do all of them at once.
-            var selectedPair = pairs.Take(20).ToArray();
-            var pairDetails = new List<string>();
-
-            foreach (var pair in selectedPair)
+            var results = new List<string>();
+            foreach (var pair in pairs.ToArray())
             {
                 pair.Check();
-
                 if (pair.Success)
                 {
-                    pairDetails.AddRange(pair.GetResultMessages());
+                    results.AddRange(pair.GetResultMessages());
                     pairs.Remove(pair);
                 }
             }
-
-            Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
+            Log($"Connections successful:{Nl}{string.Join(Nl, results)}");
         }
 
         private Entry[] CreateEntries(CodexAccess[] nodes)

DistTestCore/Helpers/PeerDownloadTestHelpers.cs

@@ -1,5 +1,6 @@
 using DistTestCore.Codex;
 using Logging;
+using Utils;
 using static DistTestCore.Helpers.FullConnectivityHelper;
 
 namespace DistTestCore.Helpers

KubernetesWorkflow/ByteSizeExtensions.cs

@@ -0,0 +1,41 @@
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public static class ByteSizeExtensions
+    {
+        public static string ToSuffixNotation(this ByteSize b)
+        {
+            long x = 1024;
+            var map = new Dictionary<long, string>
+            {
+                { Pow(x, 4), "Ti" },
+                { Pow(x, 3), "Gi" },
+                { Pow(x, 2), "Mi" },
+                { (x), "Ki" },
+            };
+
+            var bytes = b.SizeInBytes;
+            foreach (var pair in map)
+            {
+                if (bytes > pair.Key)
+                {
+                    double bytesD = bytes;
+                    double divD = pair.Key;
+                    double numD = Math.Ceiling(bytesD / divD);
+                    var v = Convert.ToInt64(numD);
+                    return $"{v}{pair.Value}";
+                }
+            }
+
+            return $"{bytes}";
+        }
+
+        private static long Pow(long x, int v)
+        {
+            long result = 1;
+            for (var i = 0; i < v; i++) result *= x;
+            return result;
+        }
+    }
+}

KubernetesWorkflow/ContainerRecipe.cs

@@ -2,26 +2,30 @@
 {
     public class ContainerRecipe
     {
-        public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, object[] additionals)
+        public ContainerRecipe(int number, string image, ContainerResources resources, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, object[] additionals)
         {
             Number = number;
             Image = image;
+            Resources = resources;
             ExposedPorts = exposedPorts;
             InternalPorts = internalPorts;
             EnvVars = envVars;
             PodLabels = podLabels;
             PodAnnotations = podAnnotations;
+            Volumes = volumes;
             Additionals = additionals;
         }
 
         public string Name { get { return $"ctnr{Number}"; } }
         public int Number { get; }
+        public ContainerResources Resources { get; }
         public string Image { get; }
         public Port[] ExposedPorts { get; }
         public Port[] InternalPorts { get; }
         public EnvVar[] EnvVars { get; }
         public PodLabels PodLabels { get; }
         public PodAnnotations PodAnnotations { get; }
+        public VolumeMount[] Volumes { get; }
         public object[] Additionals { get; }
 
         public Port GetPortByTag(string tag)
@@ -34,7 +38,9 @@
             return $"(container-recipe: {Name}, image: {Image}, " +
                 $"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " +
                 $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
-                $"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, ";
+                $"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, " +
+                $"limits: {Resources}, " +
+                $"volumes: {string.Join(",", Volumes.Select(v => $"'{v.MountPath}'"))}";
         }
     }
 
@@ -61,4 +67,18 @@
         public string Name { get; }
         public string Value { get; }
     }
+
+    public class VolumeMount
+    {
+        public VolumeMount(string volumeName, string mountPath, string resourceQuantity)
+        {
+            VolumeName = volumeName;
+            MountPath = mountPath;
+            ResourceQuantity = resourceQuantity;
+        }
+
+        public string VolumeName { get; }
+        public string MountPath { get; }
+        public string ResourceQuantity { get; }
+    }
 }

KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -1,4 +1,6 @@
-namespace KubernetesWorkflow
+using Utils;
+
+namespace KubernetesWorkflow
 {
     public abstract class ContainerRecipeFactory
     {
@@ -7,6 +9,7 @@
         private readonly List<EnvVar> envVars = new List<EnvVar>();
         private readonly PodLabels podLabels = new PodLabels();
         private readonly PodAnnotations podAnnotations = new PodAnnotations();
+        private readonly List<VolumeMount> volumeMounts = new List<VolumeMount>();
         private readonly List<object> additionals = new List<object>();
         private RecipeComponentFactory factory = null!;
 
@@ -18,12 +21,13 @@
 
             Initialize(config);
 
-            var recipe = new ContainerRecipe(containerNumber, Image,
+            var recipe = new ContainerRecipe(containerNumber, Image, Resources,
                 exposedPorts.ToArray(),
-                internalPorts.ToArray(), 
-                envVars.ToArray(), 
+                internalPorts.ToArray(),
+                envVars.ToArray(),
                 podLabels.Clone(),
                 podAnnotations.Clone(),
+                volumeMounts.ToArray(),
                 additionals.ToArray());
 
             exposedPorts.Clear();
@@ -31,6 +35,7 @@
             envVars.Clear();
             podLabels.Clear();
             podAnnotations.Clear();
+            volumeMounts.Clear();
             additionals.Clear();
             this.factory = null!;
 
@@ -39,6 +44,7 @@
 
         public abstract string AppName { get; }
         public abstract string Image { get; }
+        public ContainerResources Resources { get; } = new ContainerResources();
         protected int ContainerNumber { get; private set; } = 0;
         protected int Index { get; private set; } = 0;
         protected abstract void Initialize(StartupConfig config);
@@ -94,6 +100,14 @@
             podAnnotations.Add(name, value);
         }
 
+        protected void AddVolume(string mountPath, ByteSize volumeSize)
+        {
+            volumeMounts.Add(new VolumeMount(
+                $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}",
+                mountPath,
+                volumeSize.ToSuffixNotation()));
+        }
+
         protected void Additional(object userData)
         {
             additionals.Add(userData);

KubernetesWorkflow/ContainerResources.cs

@@ -0,0 +1,52 @@
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public class ContainerResources
+    {
+        public ContainerResourceSet Requests { get; set; } = new ContainerResourceSet();
+        public ContainerResourceSet Limits { get; set; } = new ContainerResourceSet();
+
+        public override string ToString()
+        {
+            return $"requests:{Requests}, limits:{Limits}";
+        }
+    }
+
+    public class ContainerResourceSet
+    {
+        public ContainerResourceSet(int milliCPUs, ByteSize memory)
+        {
+            MilliCPUs = milliCPUs;
+            Memory = memory;
+        }
+
+        public ContainerResourceSet(int milliCPUs)
+            : this(milliCPUs, new ByteSize(0))
+        {
+        }
+
+        public ContainerResourceSet(ByteSize memory)
+            : this(0, memory)
+        {
+        }
+
+        public ContainerResourceSet()
+            : this(0)
+        {
+        }
+
+        public int MilliCPUs { get; }
+        public ByteSize Memory { get; }
+
+        public override string ToString()
+        {
+            var result = new List<string>();
+            if (MilliCPUs == 0) result.Add("cpu: unlimited");
+            else result.Add($"cpu: {MilliCPUs} milliCPUs");
+            if (Memory.SizeInBytes == 0) result.Add("memory: unlimited");
+            else result.Add($"memory: {Memory}");
+            return string.Join(", ", result);
+        }
+    }
+}

KubernetesWorkflow/K8sController.cs

@@ -345,7 +345,8 @@ namespace KubernetesWorkflow
                         Spec = new V1PodSpec
                         {
                             NodeSelector = CreateNodeSelector(location),
-                            Containers = CreateDeploymentContainers(containerRecipes)
+                            Containers = CreateDeploymentContainers(containerRecipes),
+                            Volumes = CreateVolumes(containerRecipes)
                         }
                     }
                 }
@@ -407,7 +408,7 @@ namespace KubernetesWorkflow
 
         private List<V1Container> CreateDeploymentContainers(ContainerRecipe[] containerRecipes)
         {
-            return containerRecipes.Select(r => CreateDeploymentContainer(r)).ToList();
+            return containerRecipes.Select(CreateDeploymentContainer).ToList();
         }
 
         private V1Container CreateDeploymentContainer(ContainerRecipe recipe)
@@ -418,7 +419,91 @@ namespace KubernetesWorkflow
                 Image = recipe.Image,
                 ImagePullPolicy = "Always",
                 Ports = CreateContainerPorts(recipe),
-                Env = CreateEnv(recipe)
+                Env = CreateEnv(recipe),
+                VolumeMounts = CreateContainerVolumeMounts(recipe),
+                Resources = CreateResourceLimits(recipe)
+            };
+        }
+
+        private V1ResourceRequirements CreateResourceLimits(ContainerRecipe recipe)
+        {
+            return new V1ResourceRequirements
+            {
+                Requests = CreateResourceQuantities(recipe.Resources.Requests),
+                Limits = CreateResourceQuantities(recipe.Resources.Limits)
+            };
+        }
+
+        private Dictionary<string, ResourceQuantity> CreateResourceQuantities(ContainerResourceSet set)
+        {
+            var result = new Dictionary<string, ResourceQuantity>();
+            if (set.MilliCPUs != 0)
+            {
+                result.Add("cpu", new ResourceQuantity($"{set.MilliCPUs}m"));
+            }
+            if (set.Memory.SizeInBytes != 0)
+            {
+                result.Add("memory", new ResourceQuantity(set.Memory.ToSuffixNotation()));
+            }
+            return result;
+        }
+
+        private List<V1VolumeMount> CreateContainerVolumeMounts(ContainerRecipe recipe)
+        {
+            return recipe.Volumes.Select(CreateContainerVolumeMount).ToList();
+        }
+
+        private V1VolumeMount CreateContainerVolumeMount(VolumeMount v)
+        {
+            return new V1VolumeMount
+            {
+                Name = v.VolumeName,
+                MountPath = v.MountPath
+            };
+        }
+
+        private List<V1Volume> CreateVolumes(ContainerRecipe[] containerRecipes)
+        {
+            return containerRecipes.Where(c => c.Volumes.Any()).SelectMany(CreateVolumes).ToList();
+        }
+
+        private List<V1Volume> CreateVolumes(ContainerRecipe recipe)
+        {
+            return recipe.Volumes.Select(CreateVolume).ToList();
+        }
+
+        private V1Volume CreateVolume(VolumeMount v)
+        {
+            client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim
+            {
+                ApiVersion = "v1",
+                Metadata = new V1ObjectMeta
+                {
+                    Name = v.VolumeName
+                },
+                Spec = new V1PersistentVolumeClaimSpec
+                {
+                    AccessModes = new List<string>
+                    {
+                        "ReadWriteOnce"
+                    },
+                    Resources = new V1ResourceRequirements
+                    {
+                        Requests = new Dictionary<string, ResourceQuantity>
+                        {
+                            {"storage", new ResourceQuantity(v.ResourceQuantity) }
+                        }
+                    }
+                }
+            }, K8sTestNamespace));
+
+            return new V1Volume
+            {
+                Name = v.VolumeName,
+                PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource
+                {
+                    ClaimName = v.VolumeName
+                }
             };
         }
 

LongTests/BasicTests/DownloadTests.cs

@@ -1,5 +1,6 @@
 using DistTestCore;
 using NUnit.Framework;
+using Utils;
 
 namespace TestsLong.BasicTests
 {

LongTests/BasicTests/LargeFileTests.cs

@@ -2,6 +2,7 @@
 using DistTestCore.Codex;
 using NUnit.Framework;
 using NUnit.Framework.Interfaces;
+using Utils;
 
 namespace TestsLong.BasicTests
 {

LongTests/BasicTests/UploadTests.cs

@@ -1,5 +1,6 @@
 using DistTestCore;
 using NUnit.Framework;
+using Utils;
 
 namespace TestsLong.BasicTests
 {

LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs

@@ -1,5 +1,6 @@
 using DistTestCore;
 using NUnit.Framework;
+using Utils;
 
 namespace TestsLong.DownloadConnectivityTests
 {

Tests/BasicTests/ExampleTests.cs

@@ -1,5 +1,6 @@
 using DistTestCore;
 using NUnit.Framework;
+using Utils;
 
 namespace Tests.BasicTests
 {

Tests/BasicTests/OneClientTests.cs

@@ -1,5 +1,6 @@
 using DistTestCore;
 using NUnit.Framework;
+using Utils;
 
 namespace Tests.BasicTests
 {

Tests/BasicTests/ThreeClientTest.cs

@@ -1,5 +1,6 @@
 using DistTestCore;
 using NUnit.Framework;
+using Utils;
 
 namespace Tests.BasicTests
 {

Tests/BasicTests/TwoClientTests.cs

@@ -1,6 +1,7 @@
 using DistTestCore;
 using KubernetesWorkflow;
 using NUnit.Framework;
+using Utils;
 
 namespace Tests.BasicTests
 {

Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -1,5 +1,6 @@
 using DistTestCore;
 using NUnit.Framework;
+using Utils;
 
 namespace Tests.DownloadConnectivityTests
 {
@@ -25,8 +26,8 @@ namespace Tests.DownloadConnectivityTests
         [Test]
         [Combinatorial]
         public void FullyConnectedDownloadTest(
-            [Values(1, 3, 5)] int numberOfNodes,
-            [Values(1, 10)] int sizeMBs)
+            [Values(3, 5)] int numberOfNodes,
+            [Values(10, 80)] int sizeMBs)
         {
             SetupCodexNodes(numberOfNodes);
 

Utils/ByteSize.cs

@@ -1,10 +1,7 @@
-using Utils;
-
-namespace DistTestCore
+namespace Utils
 {
     public class ByteSize
     {
-
         public ByteSize(long sizeInBytes)
         {
             if (sizeInBytes < 0) throw new ArgumentException("Cannot create ByteSize object with size less than 0. Was: " + sizeInBytes);
@@ -18,6 +15,13 @@ namespace DistTestCore
             return SizeInBytes / (1024 * 1024);
         }
 
+        public ByteSize Multiply(double factor)
+        {
+            double bytes = SizeInBytes;
+            double result = Math.Round(bytes * factor);
+            return new ByteSize(Convert.ToInt64(result));
+        }
+
         public override bool Equals(object? obj)
         {
             return obj is ByteSize size && SizeInBytes == size.SizeInBytes;