diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 8e861f3..c6b3850 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -2,6 +2,7 @@ using DistTestCore.Codex; using DistTestCore.Marketplace; using KubernetesWorkflow; +using Utils; namespace CodexNetDeployer { diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 9b6c103..eb329fc 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -2,6 +2,7 @@ using DistTestCore.Codex; using KubernetesWorkflow; using Logging; +using Utils; namespace CodexNetDeployer { diff --git a/ContinuousTests/Tests/HoldMyBeerTest.cs b/ContinuousTests/Tests/HoldMyBeerTest.cs index 6db4d9b..0ec268f 100644 --- a/ContinuousTests/Tests/HoldMyBeerTest.cs +++ b/ContinuousTests/Tests/HoldMyBeerTest.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace ContinuousTests.Tests { diff --git a/ContinuousTests/Tests/TwoClientTest.cs b/ContinuousTests/Tests/TwoClientTest.cs index fe44816..e179944 100644 --- a/ContinuousTests/Tests/TwoClientTest.cs +++ b/ContinuousTests/Tests/TwoClientTest.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace ContinuousTests.Tests { diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index aa67789..c928b3e 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -1,5 +1,6 @@ using DistTestCore.Marketplace; using KubernetesWorkflow; +using Utils; namespace DistTestCore.Codex { @@ -16,10 +17,13 @@ namespace DistTestCore.Codex public override string AppName => "codex"; public override string Image { get; } - + public CodexContainerRecipe() { Image = GetDockerImage(); + + Resources.Requests = new ContainerResourceSet(milliCPUs: 1000, memory: 6.GB()); + Resources.Limits = new ContainerResourceSet(milliCPUs: 4000, memory: 12.GB()); } protected override void InitializeRecipe(StartupConfig startupConfig) @@ -29,7 +33,10 @@ namespace DistTestCore.Codex AddExposedPortAndVar("CODEX_API_PORT"); AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0"); - AddEnvVar("CODEX_DATA_DIR", $"datadir{ContainerNumber}"); + var dataDir = $"datadir{ContainerNumber}"; + AddEnvVar("CODEX_DATA_DIR", dataDir); + AddVolume($"codex/{dataDir}", GetVolumeCapacity(config)); + AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag); AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); @@ -91,6 +98,13 @@ namespace DistTestCore.Codex } } + private ByteSize GetVolumeCapacity(CodexStartupConfig config) + { + if (config.StorageQuota != null) return config.StorageQuota; + // Default Codex quota: 8 Gb, using +20% to be safe. + return 8.GB().Multiply(1.2); + } + private int GetAccountIndex(MarketplaceInitialConfig marketplaceConfig) { if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value; diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index c09f066..36e4757 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -1,6 +1,7 @@ using DistTestCore.Marketplace; using DistTestCore.Metrics; using KubernetesWorkflow; +using Utils; namespace DistTestCore.Codex { diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index da5fa64..f2ec51e 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -1,6 +1,7 @@ using DistTestCore.Codex; using DistTestCore.Marketplace; using KubernetesWorkflow; +using Utils; namespace DistTestCore { diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 9e13ca7..155a24f 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -7,6 +7,7 @@ using KubernetesWorkflow; using Logging; using NUnit.Framework; using System.Reflection; +using Utils; namespace DistTestCore { diff --git a/DistTestCore/GrafanaStarter.cs b/DistTestCore/GrafanaStarter.cs index 978c0f4..d4a3cd8 100644 --- a/DistTestCore/GrafanaStarter.cs +++ b/DistTestCore/GrafanaStarter.cs @@ -3,6 +3,7 @@ using IdentityModel.Client; using KubernetesWorkflow; using Newtonsoft.Json; using System.Reflection; +using Utils; namespace DistTestCore { diff --git a/DistTestCore/Helpers/FullConnectivityHelper.cs b/DistTestCore/Helpers/FullConnectivityHelper.cs index 68dfdbb..b69af0c 100644 --- a/DistTestCore/Helpers/FullConnectivityHelper.cs +++ b/DistTestCore/Helpers/FullConnectivityHelper.cs @@ -1,7 +1,6 @@ using DistTestCore.Codex; using Logging; using NUnit.Framework; -using Utils; namespace DistTestCore.Helpers { @@ -35,10 +34,9 @@ namespace DistTestCore.Helpers var entries = CreateEntries(nodes); var pairs = CreatePairs(entries); - RetryWhilePairs(pairs, () => - { - CheckAndRemoveSuccessful(pairs); - }); + // Each pair gets two chances. + CheckAndRemoveSuccessful(pairs); + CheckAndRemoveSuccessful(pairs); if (pairs.Any()) { @@ -54,35 +52,19 @@ namespace DistTestCore.Helpers } } - private static void RetryWhilePairs(List pairs, Action action) - { - var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2); - while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow) - { - action(); - - Time.Sleep(TimeSpan.FromSeconds(2)); - } - } - private void CheckAndRemoveSuccessful(List pairs) { - // For large sets, don't try and do all of them at once. - var selectedPair = pairs.Take(20).ToArray(); - var pairDetails = new List(); - - foreach (var pair in selectedPair) + var results = new List(); + foreach (var pair in pairs.ToArray()) { pair.Check(); - if (pair.Success) { - pairDetails.AddRange(pair.GetResultMessages()); + results.AddRange(pair.GetResultMessages()); pairs.Remove(pair); } } - - Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}"); + Log($"Connections successful:{Nl}{string.Join(Nl, results)}"); } private Entry[] CreateEntries(CodexAccess[] nodes) diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs index cd12e2b..c2af415 100644 --- a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using Logging; +using Utils; using static DistTestCore.Helpers.FullConnectivityHelper; namespace DistTestCore.Helpers diff --git a/KubernetesWorkflow/ByteSizeExtensions.cs b/KubernetesWorkflow/ByteSizeExtensions.cs new file mode 100644 index 0000000..c3cca24 --- /dev/null +++ b/KubernetesWorkflow/ByteSizeExtensions.cs @@ -0,0 +1,41 @@ +using Utils; + +namespace KubernetesWorkflow +{ + public static class ByteSizeExtensions + { + public static string ToSuffixNotation(this ByteSize b) + { + long x = 1024; + var map = new Dictionary + { + { Pow(x, 4), "Ti" }, + { Pow(x, 3), "Gi" }, + { Pow(x, 2), "Mi" }, + { (x), "Ki" }, + }; + + var bytes = b.SizeInBytes; + foreach (var pair in map) + { + if (bytes > pair.Key) + { + double bytesD = bytes; + double divD = pair.Key; + double numD = Math.Ceiling(bytesD / divD); + var v = Convert.ToInt64(numD); + return $"{v}{pair.Value}"; + } + } + + return $"{bytes}"; + } + + private static long Pow(long x, int v) + { + long result = 1; + for (var i = 0; i < v; i++) result *= x; + return result; + } + } +} diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index e3bd7b8..51ea2e1 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -2,26 +2,30 @@ { public class ContainerRecipe { - public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, object[] additionals) + public ContainerRecipe(int number, string image, ContainerResources resources, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, object[] additionals) { Number = number; Image = image; + Resources = resources; ExposedPorts = exposedPorts; InternalPorts = internalPorts; EnvVars = envVars; PodLabels = podLabels; PodAnnotations = podAnnotations; + Volumes = volumes; Additionals = additionals; } public string Name { get { return $"ctnr{Number}"; } } public int Number { get; } + public ContainerResources Resources { get; } public string Image { get; } public Port[] ExposedPorts { get; } public Port[] InternalPorts { get; } public EnvVar[] EnvVars { get; } public PodLabels PodLabels { get; } public PodAnnotations PodAnnotations { get; } + public VolumeMount[] Volumes { get; } public object[] Additionals { get; } public Port GetPortByTag(string tag) @@ -34,7 +38,9 @@ return $"(container-recipe: {Name}, image: {Image}, " + $"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " + $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " + - $"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, "; + $"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, " + + $"limits: {Resources}, " + + $"volumes: {string.Join(",", Volumes.Select(v => $"'{v.MountPath}'"))}"; } } @@ -61,4 +67,18 @@ public string Name { get; } public string Value { get; } } + + public class VolumeMount + { + public VolumeMount(string volumeName, string mountPath, string resourceQuantity) + { + VolumeName = volumeName; + MountPath = mountPath; + ResourceQuantity = resourceQuantity; + } + + public string VolumeName { get; } + public string MountPath { get; } + public string ResourceQuantity { get; } + } } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 2c64efb..f8c88ef 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -1,4 +1,6 @@ -namespace KubernetesWorkflow +using Utils; + +namespace KubernetesWorkflow { public abstract class ContainerRecipeFactory { @@ -7,6 +9,7 @@ private readonly List envVars = new List(); private readonly PodLabels podLabels = new PodLabels(); private readonly PodAnnotations podAnnotations = new PodAnnotations(); + private readonly List volumeMounts = new List(); private readonly List additionals = new List(); private RecipeComponentFactory factory = null!; @@ -18,12 +21,13 @@ Initialize(config); - var recipe = new ContainerRecipe(containerNumber, Image, + var recipe = new ContainerRecipe(containerNumber, Image, Resources, exposedPorts.ToArray(), - internalPorts.ToArray(), - envVars.ToArray(), + internalPorts.ToArray(), + envVars.ToArray(), podLabels.Clone(), podAnnotations.Clone(), + volumeMounts.ToArray(), additionals.ToArray()); exposedPorts.Clear(); @@ -31,6 +35,7 @@ envVars.Clear(); podLabels.Clear(); podAnnotations.Clear(); + volumeMounts.Clear(); additionals.Clear(); this.factory = null!; @@ -39,6 +44,7 @@ public abstract string AppName { get; } public abstract string Image { get; } + public ContainerResources Resources { get; } = new ContainerResources(); protected int ContainerNumber { get; private set; } = 0; protected int Index { get; private set; } = 0; protected abstract void Initialize(StartupConfig config); @@ -94,6 +100,14 @@ podAnnotations.Add(name, value); } + protected void AddVolume(string mountPath, ByteSize volumeSize) + { + volumeMounts.Add(new VolumeMount( + $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}", + mountPath, + volumeSize.ToSuffixNotation())); + } + protected void Additional(object userData) { additionals.Add(userData); diff --git a/KubernetesWorkflow/ContainerResources.cs b/KubernetesWorkflow/ContainerResources.cs new file mode 100644 index 0000000..40e5dc5 --- /dev/null +++ b/KubernetesWorkflow/ContainerResources.cs @@ -0,0 +1,52 @@ +using Utils; + +namespace KubernetesWorkflow +{ + public class ContainerResources + { + public ContainerResourceSet Requests { get; set; } = new ContainerResourceSet(); + public ContainerResourceSet Limits { get; set; } = new ContainerResourceSet(); + + public override string ToString() + { + return $"requests:{Requests}, limits:{Limits}"; + } + } + + public class ContainerResourceSet + { + public ContainerResourceSet(int milliCPUs, ByteSize memory) + { + MilliCPUs = milliCPUs; + Memory = memory; + } + + public ContainerResourceSet(int milliCPUs) + : this(milliCPUs, new ByteSize(0)) + { + } + + public ContainerResourceSet(ByteSize memory) + : this(0, memory) + { + } + + public ContainerResourceSet() + : this(0) + { + } + + public int MilliCPUs { get; } + public ByteSize Memory { get; } + + public override string ToString() + { + var result = new List(); + if (MilliCPUs == 0) result.Add("cpu: unlimited"); + else result.Add($"cpu: {MilliCPUs} milliCPUs"); + if (Memory.SizeInBytes == 0) result.Add("memory: unlimited"); + else result.Add($"memory: {Memory}"); + return string.Join(", ", result); + } + } +} diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index d06070d..c162bcf 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -345,7 +345,8 @@ namespace KubernetesWorkflow Spec = new V1PodSpec { NodeSelector = CreateNodeSelector(location), - Containers = CreateDeploymentContainers(containerRecipes) + Containers = CreateDeploymentContainers(containerRecipes), + Volumes = CreateVolumes(containerRecipes) } } } @@ -407,7 +408,7 @@ namespace KubernetesWorkflow private List CreateDeploymentContainers(ContainerRecipe[] containerRecipes) { - return containerRecipes.Select(r => CreateDeploymentContainer(r)).ToList(); + return containerRecipes.Select(CreateDeploymentContainer).ToList(); } private V1Container CreateDeploymentContainer(ContainerRecipe recipe) @@ -418,7 +419,91 @@ namespace KubernetesWorkflow Image = recipe.Image, ImagePullPolicy = "Always", Ports = CreateContainerPorts(recipe), - Env = CreateEnv(recipe) + Env = CreateEnv(recipe), + VolumeMounts = CreateContainerVolumeMounts(recipe), + Resources = CreateResourceLimits(recipe) + }; + } + + private V1ResourceRequirements CreateResourceLimits(ContainerRecipe recipe) + { + return new V1ResourceRequirements + { + Requests = CreateResourceQuantities(recipe.Resources.Requests), + Limits = CreateResourceQuantities(recipe.Resources.Limits) + }; + } + + private Dictionary CreateResourceQuantities(ContainerResourceSet set) + { + var result = new Dictionary(); + if (set.MilliCPUs != 0) + { + result.Add("cpu", new ResourceQuantity($"{set.MilliCPUs}m")); + } + if (set.Memory.SizeInBytes != 0) + { + result.Add("memory", new ResourceQuantity(set.Memory.ToSuffixNotation())); + } + return result; + } + + private List CreateContainerVolumeMounts(ContainerRecipe recipe) + { + return recipe.Volumes.Select(CreateContainerVolumeMount).ToList(); + } + + private V1VolumeMount CreateContainerVolumeMount(VolumeMount v) + { + return new V1VolumeMount + { + Name = v.VolumeName, + MountPath = v.MountPath + }; + } + + private List CreateVolumes(ContainerRecipe[] containerRecipes) + { + return containerRecipes.Where(c => c.Volumes.Any()).SelectMany(CreateVolumes).ToList(); + } + + private List CreateVolumes(ContainerRecipe recipe) + { + return recipe.Volumes.Select(CreateVolume).ToList(); + } + + private V1Volume CreateVolume(VolumeMount v) + { + client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = v.VolumeName + }, + Spec = new V1PersistentVolumeClaimSpec + { + AccessModes = new List + { + "ReadWriteOnce" + }, + Resources = new V1ResourceRequirements + { + Requests = new Dictionary + { + {"storage", new ResourceQuantity(v.ResourceQuantity) } + } + } + } + }, K8sTestNamespace)); + + return new V1Volume + { + Name = v.VolumeName, + PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource + { + ClaimName = v.VolumeName + } }; } diff --git a/LongTests/BasicTests/DownloadTests.cs b/LongTests/BasicTests/DownloadTests.cs index 5034462..5e01e3c 100644 --- a/LongTests/BasicTests/DownloadTests.cs +++ b/LongTests/BasicTests/DownloadTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace TestsLong.BasicTests { diff --git a/LongTests/BasicTests/LargeFileTests.cs b/LongTests/BasicTests/LargeFileTests.cs index 9eb3a89..2d834e5 100644 --- a/LongTests/BasicTests/LargeFileTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -2,6 +2,7 @@ using DistTestCore.Codex; using NUnit.Framework; using NUnit.Framework.Interfaces; +using Utils; namespace TestsLong.BasicTests { diff --git a/LongTests/BasicTests/UploadTests.cs b/LongTests/BasicTests/UploadTests.cs index ab005fe..69823eb 100644 --- a/LongTests/BasicTests/UploadTests.cs +++ b/LongTests/BasicTests/UploadTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace TestsLong.BasicTests { diff --git a/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs b/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs index d0650f6..6de5c38 100644 --- a/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs +++ b/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace TestsLong.DownloadConnectivityTests { diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 58d4cbc..1492ce1 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace Tests.BasicTests { diff --git a/Tests/BasicTests/OneClientTests.cs b/Tests/BasicTests/OneClientTests.cs index b22e53b..a31be3c 100644 --- a/Tests/BasicTests/OneClientTests.cs +++ b/Tests/BasicTests/OneClientTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace Tests.BasicTests { diff --git a/Tests/BasicTests/ThreeClientTest.cs b/Tests/BasicTests/ThreeClientTest.cs index 78ef25e..c857e35 100644 --- a/Tests/BasicTests/ThreeClientTest.cs +++ b/Tests/BasicTests/ThreeClientTest.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace Tests.BasicTests { diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs index 12bab13..4bebc20 100644 --- a/Tests/BasicTests/TwoClientTests.cs +++ b/Tests/BasicTests/TwoClientTests.cs @@ -1,6 +1,7 @@ using DistTestCore; using KubernetesWorkflow; using NUnit.Framework; +using Utils; namespace Tests.BasicTests { diff --git a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index 7cf546b..9fb5630 100644 --- a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace Tests.DownloadConnectivityTests { @@ -25,8 +26,8 @@ namespace Tests.DownloadConnectivityTests [Test] [Combinatorial] public void FullyConnectedDownloadTest( - [Values(1, 3, 5)] int numberOfNodes, - [Values(1, 10)] int sizeMBs) + [Values(3, 5)] int numberOfNodes, + [Values(10, 80)] int sizeMBs) { SetupCodexNodes(numberOfNodes); diff --git a/DistTestCore/ByteSize.cs b/Utils/ByteSize.cs similarity index 89% rename from DistTestCore/ByteSize.cs rename to Utils/ByteSize.cs index bbf01cc..b069b04 100644 --- a/DistTestCore/ByteSize.cs +++ b/Utils/ByteSize.cs @@ -1,10 +1,7 @@ -using Utils; - -namespace DistTestCore +namespace Utils { public class ByteSize { - public ByteSize(long sizeInBytes) { if (sizeInBytes < 0) throw new ArgumentException("Cannot create ByteSize object with size less than 0. Was: " + sizeInBytes); @@ -18,6 +15,13 @@ namespace DistTestCore return SizeInBytes / (1024 * 1024); } + public ByteSize Multiply(double factor) + { + double bytes = SizeInBytes; + double result = Math.Round(bytes * factor); + return new ByteSize(Convert.ToInt64(result)); + } + public override bool Equals(object? obj) { return obj is ByteSize size && SizeInBytes == size.SizeInBytes;