From aa3e500a581bd051f27875a792ac3f43ac7214e9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 7 Sep 2023 08:19:19 +0200 Subject: [PATCH 01/12] Adds volume support to container recipe factory --- DistTestCore/Codex/CodexContainerRecipe.cs | 5 ++++- KubernetesWorkflow/ContainerRecipe.cs | 19 +++++++++++++++++-- KubernetesWorkflow/ContainerRecipeFactory.cs | 14 ++++++++++++-- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index aa67789..1b03a50 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -29,7 +29,10 @@ namespace DistTestCore.Codex AddExposedPortAndVar("CODEX_API_PORT"); AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0"); - AddEnvVar("CODEX_DATA_DIR", $"datadir{ContainerNumber}"); + var dataDir = $"datadir{ContainerNumber}"; + AddEnvVar("CODEX_DATA_DIR", dataDir); + AddVolume(dataDir); + AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag); AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index e3bd7b8..f58c3fd 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -2,7 +2,7 @@ { public class ContainerRecipe { - public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, object[] additionals) + public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, object[] additionals) { Number = number; Image = image; @@ -11,6 +11,7 @@ EnvVars = envVars; PodLabels = podLabels; PodAnnotations = podAnnotations; + Volumes = volumes; Additionals = additionals; } @@ -22,6 +23,7 @@ public EnvVar[] EnvVars { get; } public PodLabels PodLabels { get; } public PodAnnotations PodAnnotations { get; } + public VolumeMount[] Volumes { get; } public object[] Additionals { get; } public Port GetPortByTag(string tag) @@ -34,7 +36,8 @@ return $"(container-recipe: {Name}, image: {Image}, " + $"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " + $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " + - $"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, "; + $"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, " + + $"volumes: {string.Join(",", Volumes.Select(v => $"'{v.MountPath}'"))}"; } } @@ -61,4 +64,16 @@ public string Name { get; } public string Value { get; } } + + public class VolumeMount + { + public VolumeMount(string volumeName, string mountPath) + { + VolumeName = volumeName; + MountPath = mountPath; + } + + public string VolumeName { get; } + public string MountPath { get; } + } } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index 2c64efb..f10d062 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -7,6 +7,7 @@ private readonly List envVars = new List(); private readonly PodLabels podLabels = new PodLabels(); private readonly PodAnnotations podAnnotations = new PodAnnotations(); + private readonly List volumeMounts = new List(); private readonly List additionals = new List(); private RecipeComponentFactory factory = null!; @@ -20,10 +21,11 @@ var recipe = new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), - internalPorts.ToArray(), - envVars.ToArray(), + internalPorts.ToArray(), + envVars.ToArray(), podLabels.Clone(), podAnnotations.Clone(), + volumeMounts.ToArray(), additionals.ToArray()); exposedPorts.Clear(); @@ -31,6 +33,7 @@ envVars.Clear(); podLabels.Clear(); podAnnotations.Clear(); + volumeMounts.Clear(); additionals.Clear(); this.factory = null!; @@ -94,6 +97,13 @@ podAnnotations.Add(name, value); } + protected void AddVolume(string mountPath) + { + volumeMounts.Add(new VolumeMount( + $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}", + mountPath)); + } + protected void Additional(object userData) { additionals.Add(userData); From 999ed6cfc8fc5c8ad77b0eee1428a0f12bb2ea2d Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 7 Sep 2023 10:37:52 +0200 Subject: [PATCH 02/12] Creates persistent-volume-claims for codex nodes --- DistTestCore/Codex/CodexContainerRecipe.cs | 9 ++- KubernetesWorkflow/ContainerRecipe.cs | 4 +- KubernetesWorkflow/ContainerRecipeFactory.cs | 26 +++++++- KubernetesWorkflow/K8sController.cs | 70 +++++++++++++++++++- 4 files changed, 102 insertions(+), 7 deletions(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 1b03a50..b38b3e7 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -31,7 +31,7 @@ namespace DistTestCore.Codex var dataDir = $"datadir{ContainerNumber}"; AddEnvVar("CODEX_DATA_DIR", dataDir); - AddVolume(dataDir); + AddVolume($"codex/{dataDir}", GetVolumeCapacity(config)); AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag); AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); @@ -94,6 +94,13 @@ namespace DistTestCore.Codex } } + private long GetVolumeCapacity(CodexStartupConfig config) + { + if (config.StorageQuota != null) return config.StorageQuota.SizeInBytes; + // Default Codex quota: 8 Gb, using 9 to be safe. + return 9.GB().SizeInBytes; + } + private int GetAccountIndex(MarketplaceInitialConfig marketplaceConfig) { if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value; diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index f58c3fd..ed16b49 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -67,13 +67,15 @@ public class VolumeMount { - public VolumeMount(string volumeName, string mountPath) + public VolumeMount(string volumeName, string mountPath, string resourceQuantity) { VolumeName = volumeName; MountPath = mountPath; + ResourceQuantity = resourceQuantity; } public string VolumeName { get; } public string MountPath { get; } + public string ResourceQuantity { get; } } } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index f10d062..d5c75a4 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -97,16 +97,38 @@ podAnnotations.Add(name, value); } - protected void AddVolume(string mountPath) + protected void AddVolume(string mountPath, long capacityBytes) { volumeMounts.Add(new VolumeMount( $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}", - mountPath)); + mountPath, + FormatBytesQuantity(capacityBytes))); } protected void Additional(object userData) { additionals.Add(userData); } + + private static string FormatBytesQuantity(long capacityBytes) + { + var map = new Dictionary + { + { (1024*1024*1024), "Gi" }, + { (1024*1024), "Mi" }, + { (1024), "Ki" }, + }; + + foreach (var pair in map) + { + if (capacityBytes > pair.Key) + { + var v = (capacityBytes / pair.Key) + 1; + return $"{v}{pair.Value}"; + } + } + + return $"{capacityBytes * 2}Ki"; + } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index d06070d..f691f7c 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -1,6 +1,9 @@ using k8s; using k8s.Models; using Logging; +using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources; +using System.Collections.Generic; +using System.Xml.Linq; using Utils; namespace KubernetesWorkflow @@ -345,7 +348,8 @@ namespace KubernetesWorkflow Spec = new V1PodSpec { NodeSelector = CreateNodeSelector(location), - Containers = CreateDeploymentContainers(containerRecipes) + Containers = CreateDeploymentContainers(containerRecipes), + Volumes = CreateVolumes(containerRecipes) } } } @@ -407,7 +411,7 @@ namespace KubernetesWorkflow private List CreateDeploymentContainers(ContainerRecipe[] containerRecipes) { - return containerRecipes.Select(r => CreateDeploymentContainer(r)).ToList(); + return containerRecipes.Select(CreateDeploymentContainer).ToList(); } private V1Container CreateDeploymentContainer(ContainerRecipe recipe) @@ -418,7 +422,67 @@ namespace KubernetesWorkflow Image = recipe.Image, ImagePullPolicy = "Always", Ports = CreateContainerPorts(recipe), - Env = CreateEnv(recipe) + Env = CreateEnv(recipe), + VolumeMounts = CreateContainerVolumeMounts(recipe) + }; + } + + private List CreateContainerVolumeMounts(ContainerRecipe recipe) + { + return recipe.Volumes.Select(CreateContainerVolumeMount).ToList(); + } + + private V1VolumeMount CreateContainerVolumeMount(VolumeMount v) + { + return new V1VolumeMount + { + Name = v.VolumeName, + MountPath = v.MountPath + }; + } + + private List CreateVolumes(ContainerRecipe[] containerRecipes) + { + return containerRecipes.Where(c => c.Volumes.Any()).SelectMany(CreateVolumes).ToList(); + } + + private List CreateVolumes(ContainerRecipe recipe) + { + return recipe.Volumes.Select(CreateVolume).ToList(); + } + + private V1Volume CreateVolume(VolumeMount v) + { + client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = v.VolumeName + }, + Spec = new V1PersistentVolumeClaimSpec + { + AccessModes = new List + { + "ReadWriteMany" + }, + Resources = new V1ResourceRequirements + { + Requests = new Dictionary + { + {"storage", new ResourceQuantity(v.ResourceQuantity) } + } + } + } + }, K8sTestNamespace)); + + return new V1Volume + { + Name = v.VolumeName, + PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource + { + ClaimName = v.VolumeName + } }; } From 075d9e6ae2ee3b208a0164ffef1726ad446df03b Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 7 Sep 2023 10:55:35 +0200 Subject: [PATCH 03/12] switched to ReadWriteOnce access mode --- KubernetesWorkflow/K8sController.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index f691f7c..52d9300 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -464,7 +464,7 @@ namespace KubernetesWorkflow { AccessModes = new List { - "ReadWriteMany" + "ReadWriteOnce" }, Resources = new V1ResourceRequirements { From df6da29a6978aaef5462940ab7657678d4d7a62b Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 8 Sep 2023 09:39:56 +0200 Subject: [PATCH 04/12] Moves ByteSize to Utils assembly. --- DistTestCore/Codex/CodexContainerRecipe.cs | 9 ++-- DistTestCore/Codex/CodexStartupConfig.cs | 1 + KubernetesWorkflow/ByteSizeExtensions.cs | 41 +++++++++++++++++++ KubernetesWorkflow/ContainerRecipeFactory.cs | 29 +++---------- .../FullyConnectedDownloadTests.cs | 1 + {DistTestCore => Utils}/ByteSize.cs | 12 ++++-- 6 files changed, 61 insertions(+), 32 deletions(-) create mode 100644 KubernetesWorkflow/ByteSizeExtensions.cs rename {DistTestCore => Utils}/ByteSize.cs (89%) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index b38b3e7..37d1033 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -1,5 +1,6 @@ using DistTestCore.Marketplace; using KubernetesWorkflow; +using Utils; namespace DistTestCore.Codex { @@ -94,11 +95,11 @@ namespace DistTestCore.Codex } } - private long GetVolumeCapacity(CodexStartupConfig config) + private ByteSize GetVolumeCapacity(CodexStartupConfig config) { - if (config.StorageQuota != null) return config.StorageQuota.SizeInBytes; - // Default Codex quota: 8 Gb, using 9 to be safe. - return 9.GB().SizeInBytes; + if (config.StorageQuota != null) return config.StorageQuota; + // Default Codex quota: 8 Gb, using +20% to be safe. + return 8.GB().Multiply(1.2); } private int GetAccountIndex(MarketplaceInitialConfig marketplaceConfig) diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index c09f066..36e4757 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -1,6 +1,7 @@ using DistTestCore.Marketplace; using DistTestCore.Metrics; using KubernetesWorkflow; +using Utils; namespace DistTestCore.Codex { diff --git a/KubernetesWorkflow/ByteSizeExtensions.cs b/KubernetesWorkflow/ByteSizeExtensions.cs new file mode 100644 index 0000000..c3cca24 --- /dev/null +++ b/KubernetesWorkflow/ByteSizeExtensions.cs @@ -0,0 +1,41 @@ +using Utils; + +namespace KubernetesWorkflow +{ + public static class ByteSizeExtensions + { + public static string ToSuffixNotation(this ByteSize b) + { + long x = 1024; + var map = new Dictionary + { + { Pow(x, 4), "Ti" }, + { Pow(x, 3), "Gi" }, + { Pow(x, 2), "Mi" }, + { (x), "Ki" }, + }; + + var bytes = b.SizeInBytes; + foreach (var pair in map) + { + if (bytes > pair.Key) + { + double bytesD = bytes; + double divD = pair.Key; + double numD = Math.Ceiling(bytesD / divD); + var v = Convert.ToInt64(numD); + return $"{v}{pair.Value}"; + } + } + + return $"{bytes}"; + } + + private static long Pow(long x, int v) + { + long result = 1; + for (var i = 0; i < v; i++) result *= x; + return result; + } + } +} diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index d5c75a4..ad3e7de 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -1,4 +1,6 @@ -namespace KubernetesWorkflow +using Utils; + +namespace KubernetesWorkflow { public abstract class ContainerRecipeFactory { @@ -97,38 +99,17 @@ podAnnotations.Add(name, value); } - protected void AddVolume(string mountPath, long capacityBytes) + protected void AddVolume(string mountPath, ByteSize volumeSize) { volumeMounts.Add(new VolumeMount( $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}", mountPath, - FormatBytesQuantity(capacityBytes))); + volumeSize.ToSuffixNotation())); } protected void Additional(object userData) { additionals.Add(userData); } - - private static string FormatBytesQuantity(long capacityBytes) - { - var map = new Dictionary - { - { (1024*1024*1024), "Gi" }, - { (1024*1024), "Mi" }, - { (1024), "Ki" }, - }; - - foreach (var pair in map) - { - if (capacityBytes > pair.Key) - { - var v = (capacityBytes / pair.Key) + 1; - return $"{v}{pair.Value}"; - } - } - - return $"{capacityBytes * 2}Ki"; - } } } diff --git a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index 7cf546b..02ad9b0 100644 --- a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace Tests.DownloadConnectivityTests { diff --git a/DistTestCore/ByteSize.cs b/Utils/ByteSize.cs similarity index 89% rename from DistTestCore/ByteSize.cs rename to Utils/ByteSize.cs index bbf01cc..b069b04 100644 --- a/DistTestCore/ByteSize.cs +++ b/Utils/ByteSize.cs @@ -1,10 +1,7 @@ -using Utils; - -namespace DistTestCore +namespace Utils { public class ByteSize { - public ByteSize(long sizeInBytes) { if (sizeInBytes < 0) throw new ArgumentException("Cannot create ByteSize object with size less than 0. Was: " + sizeInBytes); @@ -18,6 +15,13 @@ namespace DistTestCore return SizeInBytes / (1024 * 1024); } + public ByteSize Multiply(double factor) + { + double bytes = SizeInBytes; + double result = Math.Round(bytes * factor); + return new ByteSize(Convert.ToInt64(result)); + } + public override bool Equals(object? obj) { return obj is ByteSize size && SizeInBytes == size.SizeInBytes; From 195527b650e62dbe40a6c1fdc3e40dca24cc519b Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 8 Sep 2023 10:14:52 +0200 Subject: [PATCH 05/12] Wires up resource limits --- DistTestCore/Codex/CodexContainerRecipe.cs | 1 + .../Helpers/PeerDownloadTestHelpers.cs | 1 + KubernetesWorkflow/ContainerRecipe.cs | 5 ++- KubernetesWorkflow/ContainerRecipeFactory.cs | 3 +- KubernetesWorkflow/K8sController.cs | 25 ++++++++++- KubernetesWorkflow/ResourceLimits.cs | 41 +++++++++++++++++++ 6 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 KubernetesWorkflow/ResourceLimits.cs diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 37d1033..de2ccbd 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -17,6 +17,7 @@ namespace DistTestCore.Codex public override string AppName => "codex"; public override string Image { get; } + protected override ResourceLimits ResourceLimits => new ResourceLimits(milliCPUs: 500, memory: 2.GB()); public CodexContainerRecipe() { diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs index cd12e2b..c2af415 100644 --- a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -1,5 +1,6 @@ using DistTestCore.Codex; using Logging; +using Utils; using static DistTestCore.Helpers.FullConnectivityHelper; namespace DistTestCore.Helpers diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index ed16b49..d4c7521 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -2,10 +2,11 @@ { public class ContainerRecipe { - public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, object[] additionals) + public ContainerRecipe(int number, string image, ResourceLimits resourceLimits, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, object[] additionals) { Number = number; Image = image; + ResourceLimits = resourceLimits; ExposedPorts = exposedPorts; InternalPorts = internalPorts; EnvVars = envVars; @@ -17,6 +18,7 @@ public string Name { get { return $"ctnr{Number}"; } } public int Number { get; } + public ResourceLimits ResourceLimits { get; } public string Image { get; } public Port[] ExposedPorts { get; } public Port[] InternalPorts { get; } @@ -37,6 +39,7 @@ $"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " + $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " + $"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, " + + $"limits: {ResourceLimits}, " + $"volumes: {string.Join(",", Volumes.Select(v => $"'{v.MountPath}'"))}"; } } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index ad3e7de..cc11fc6 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -21,7 +21,7 @@ namespace KubernetesWorkflow Initialize(config); - var recipe = new ContainerRecipe(containerNumber, Image, + var recipe = new ContainerRecipe(containerNumber, Image, ResourceLimits, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray(), @@ -46,6 +46,7 @@ namespace KubernetesWorkflow public abstract string Image { get; } protected int ContainerNumber { get; private set; } = 0; protected int Index { get; private set; } = 0; + protected virtual ResourceLimits ResourceLimits { get; } = new ResourceLimits(); protected abstract void Initialize(StartupConfig config); protected Port AddExposedPort(string tag = "") diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 52d9300..8c35c59 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -423,10 +423,33 @@ namespace KubernetesWorkflow ImagePullPolicy = "Always", Ports = CreateContainerPorts(recipe), Env = CreateEnv(recipe), - VolumeMounts = CreateContainerVolumeMounts(recipe) + VolumeMounts = CreateContainerVolumeMounts(recipe), + Resources = CreateResourceLimits(recipe) }; } + private V1ResourceRequirements CreateResourceLimits(ContainerRecipe recipe) + { + return new V1ResourceRequirements + { + Limits = CreateResourceLimit(recipe.ResourceLimits) + }; + } + + private Dictionary CreateResourceLimit(ResourceLimits limits) + { + var result = new Dictionary(); + if (limits.MilliCPUs != 0) + { + result.Add("cpu", new ResourceQuantity($"{limits.MilliCPUs}m")); + } + if (limits.Memory.SizeInBytes != 0) + { + result.Add("memory", new ResourceQuantity(limits.Memory.ToSuffixNotation())); + } + return result; + } + private List CreateContainerVolumeMounts(ContainerRecipe recipe) { return recipe.Volumes.Select(CreateContainerVolumeMount).ToList(); diff --git a/KubernetesWorkflow/ResourceLimits.cs b/KubernetesWorkflow/ResourceLimits.cs new file mode 100644 index 0000000..ed30880 --- /dev/null +++ b/KubernetesWorkflow/ResourceLimits.cs @@ -0,0 +1,41 @@ +using Utils; + +namespace KubernetesWorkflow +{ + public class ResourceLimits + { + public ResourceLimits(int milliCPUs, ByteSize memory) + { + MilliCPUs = milliCPUs; + Memory = memory; + } + + public ResourceLimits(int milliCPUs) + : this(milliCPUs, new ByteSize(0)) + { + } + + public ResourceLimits(ByteSize memory) + : this(0, memory) + { + } + + public ResourceLimits() + : this(0) + { + } + + public int MilliCPUs { get; } + public ByteSize Memory { get; } + + public override string ToString() + { + var result = new List(); + if (MilliCPUs == 0) result.Add("cpu: unlimited"); + else result.Add($"cpu: {MilliCPUs} milliCPUs"); + if (Memory.SizeInBytes == 0) result.Add("memory: unlimited"); + else result.Add($"memory: {Memory}"); + return string.Join(", ", result); + } + } +} From 13dcaff5d1cf25ffba42635d510bb8e14778cc09 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 8 Sep 2023 10:16:12 +0200 Subject: [PATCH 06/12] Reduces resources for codex instances --- DistTestCore/Codex/CodexContainerRecipe.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index de2ccbd..54c3645 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -17,7 +17,7 @@ namespace DistTestCore.Codex public override string AppName => "codex"; public override string Image { get; } - protected override ResourceLimits ResourceLimits => new ResourceLimits(milliCPUs: 500, memory: 2.GB()); + protected override ResourceLimits ResourceLimits => new ResourceLimits(milliCPUs: 250, memory: 512.MB()); public CodexContainerRecipe() { From 9d2dbbbcab7f66b9654d6eff9725c46919bf2a23 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 8 Sep 2023 10:21:40 +0200 Subject: [PATCH 07/12] Fixes missing usings --- CodexNetDeployer/CodexNodeStarter.cs | 1 + CodexNetDeployer/Deployer.cs | 1 + ContinuousTests/Tests/HoldMyBeerTest.cs | 1 + ContinuousTests/Tests/TwoClientTest.cs | 1 + DistTestCore/CodexSetup.cs | 1 + DistTestCore/DistTest.cs | 1 + DistTestCore/GrafanaStarter.cs | 1 + LongTests/BasicTests/DownloadTests.cs | 1 + LongTests/BasicTests/LargeFileTests.cs | 1 + LongTests/BasicTests/UploadTests.cs | 1 + .../DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs | 1 + Tests/BasicTests/ExampleTests.cs | 1 + Tests/BasicTests/OneClientTests.cs | 1 + Tests/BasicTests/ThreeClientTest.cs | 1 + Tests/BasicTests/TwoClientTests.cs | 1 + 15 files changed, 15 insertions(+) diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 8e861f3..c6b3850 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -2,6 +2,7 @@ using DistTestCore.Codex; using DistTestCore.Marketplace; using KubernetesWorkflow; +using Utils; namespace CodexNetDeployer { diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 9b6c103..eb329fc 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -2,6 +2,7 @@ using DistTestCore.Codex; using KubernetesWorkflow; using Logging; +using Utils; namespace CodexNetDeployer { diff --git a/ContinuousTests/Tests/HoldMyBeerTest.cs b/ContinuousTests/Tests/HoldMyBeerTest.cs index 6db4d9b..0ec268f 100644 --- a/ContinuousTests/Tests/HoldMyBeerTest.cs +++ b/ContinuousTests/Tests/HoldMyBeerTest.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace ContinuousTests.Tests { diff --git a/ContinuousTests/Tests/TwoClientTest.cs b/ContinuousTests/Tests/TwoClientTest.cs index fe44816..e179944 100644 --- a/ContinuousTests/Tests/TwoClientTest.cs +++ b/ContinuousTests/Tests/TwoClientTest.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace ContinuousTests.Tests { diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index da5fa64..f2ec51e 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -1,6 +1,7 @@ using DistTestCore.Codex; using DistTestCore.Marketplace; using KubernetesWorkflow; +using Utils; namespace DistTestCore { diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 9e13ca7..155a24f 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -7,6 +7,7 @@ using KubernetesWorkflow; using Logging; using NUnit.Framework; using System.Reflection; +using Utils; namespace DistTestCore { diff --git a/DistTestCore/GrafanaStarter.cs b/DistTestCore/GrafanaStarter.cs index 978c0f4..d4a3cd8 100644 --- a/DistTestCore/GrafanaStarter.cs +++ b/DistTestCore/GrafanaStarter.cs @@ -3,6 +3,7 @@ using IdentityModel.Client; using KubernetesWorkflow; using Newtonsoft.Json; using System.Reflection; +using Utils; namespace DistTestCore { diff --git a/LongTests/BasicTests/DownloadTests.cs b/LongTests/BasicTests/DownloadTests.cs index 5034462..5e01e3c 100644 --- a/LongTests/BasicTests/DownloadTests.cs +++ b/LongTests/BasicTests/DownloadTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace TestsLong.BasicTests { diff --git a/LongTests/BasicTests/LargeFileTests.cs b/LongTests/BasicTests/LargeFileTests.cs index 9eb3a89..2d834e5 100644 --- a/LongTests/BasicTests/LargeFileTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -2,6 +2,7 @@ using DistTestCore.Codex; using NUnit.Framework; using NUnit.Framework.Interfaces; +using Utils; namespace TestsLong.BasicTests { diff --git a/LongTests/BasicTests/UploadTests.cs b/LongTests/BasicTests/UploadTests.cs index ab005fe..69823eb 100644 --- a/LongTests/BasicTests/UploadTests.cs +++ b/LongTests/BasicTests/UploadTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace TestsLong.BasicTests { diff --git a/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs b/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs index d0650f6..6de5c38 100644 --- a/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs +++ b/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace TestsLong.DownloadConnectivityTests { diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 58d4cbc..1492ce1 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace Tests.BasicTests { diff --git a/Tests/BasicTests/OneClientTests.cs b/Tests/BasicTests/OneClientTests.cs index b22e53b..a31be3c 100644 --- a/Tests/BasicTests/OneClientTests.cs +++ b/Tests/BasicTests/OneClientTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace Tests.BasicTests { diff --git a/Tests/BasicTests/ThreeClientTest.cs b/Tests/BasicTests/ThreeClientTest.cs index 78ef25e..c857e35 100644 --- a/Tests/BasicTests/ThreeClientTest.cs +++ b/Tests/BasicTests/ThreeClientTest.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace Tests.BasicTests { diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs index 12bab13..4bebc20 100644 --- a/Tests/BasicTests/TwoClientTests.cs +++ b/Tests/BasicTests/TwoClientTests.cs @@ -1,6 +1,7 @@ using DistTestCore; using KubernetesWorkflow; using NUnit.Framework; +using Utils; namespace Tests.BasicTests { From 75b16bd2ca9ea6832b8994d6cf03ff4f49a3cc6e Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 8 Sep 2023 13:47:49 +0200 Subject: [PATCH 08/12] Include request resource sets. --- DistTestCore/Codex/CodexContainerRecipe.cs | 6 ++++-- KubernetesWorkflow/ContainerRecipe.cs | 8 +++---- KubernetesWorkflow/ContainerRecipeFactory.cs | 4 ++-- ...esourceLimits.cs => ContainerResources.cs} | 21 ++++++++++++++----- KubernetesWorkflow/K8sController.cs | 13 ++++++------ 5 files changed, 33 insertions(+), 19 deletions(-) rename KubernetesWorkflow/{ResourceLimits.cs => ContainerResources.cs} (56%) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 54c3645..c9c0096 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -17,11 +17,13 @@ namespace DistTestCore.Codex public override string AppName => "codex"; public override string Image { get; } - protected override ResourceLimits ResourceLimits => new ResourceLimits(milliCPUs: 250, memory: 512.MB()); - + public CodexContainerRecipe() { Image = GetDockerImage(); + + Resources.Requests = new ContainerResourceSet(milliCPUs: 250, memory: 512.MB()); + Resources.Limits = new ContainerResourceSet(milliCPUs: 1000, memory: 2.GB()); } protected override void InitializeRecipe(StartupConfig startupConfig) diff --git a/KubernetesWorkflow/ContainerRecipe.cs b/KubernetesWorkflow/ContainerRecipe.cs index d4c7521..51ea2e1 100644 --- a/KubernetesWorkflow/ContainerRecipe.cs +++ b/KubernetesWorkflow/ContainerRecipe.cs @@ -2,11 +2,11 @@ { public class ContainerRecipe { - public ContainerRecipe(int number, string image, ResourceLimits resourceLimits, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, object[] additionals) + public ContainerRecipe(int number, string image, ContainerResources resources, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, object[] additionals) { Number = number; Image = image; - ResourceLimits = resourceLimits; + Resources = resources; ExposedPorts = exposedPorts; InternalPorts = internalPorts; EnvVars = envVars; @@ -18,7 +18,7 @@ public string Name { get { return $"ctnr{Number}"; } } public int Number { get; } - public ResourceLimits ResourceLimits { get; } + public ContainerResources Resources { get; } public string Image { get; } public Port[] ExposedPorts { get; } public Port[] InternalPorts { get; } @@ -39,7 +39,7 @@ $"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " + $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " + $"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, " + - $"limits: {ResourceLimits}, " + + $"limits: {Resources}, " + $"volumes: {string.Join(",", Volumes.Select(v => $"'{v.MountPath}'"))}"; } } diff --git a/KubernetesWorkflow/ContainerRecipeFactory.cs b/KubernetesWorkflow/ContainerRecipeFactory.cs index cc11fc6..f8c88ef 100644 --- a/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -21,7 +21,7 @@ namespace KubernetesWorkflow Initialize(config); - var recipe = new ContainerRecipe(containerNumber, Image, ResourceLimits, + var recipe = new ContainerRecipe(containerNumber, Image, Resources, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray(), @@ -44,9 +44,9 @@ namespace KubernetesWorkflow public abstract string AppName { get; } public abstract string Image { get; } + public ContainerResources Resources { get; } = new ContainerResources(); protected int ContainerNumber { get; private set; } = 0; protected int Index { get; private set; } = 0; - protected virtual ResourceLimits ResourceLimits { get; } = new ResourceLimits(); protected abstract void Initialize(StartupConfig config); protected Port AddExposedPort(string tag = "") diff --git a/KubernetesWorkflow/ResourceLimits.cs b/KubernetesWorkflow/ContainerResources.cs similarity index 56% rename from KubernetesWorkflow/ResourceLimits.cs rename to KubernetesWorkflow/ContainerResources.cs index ed30880..40e5dc5 100644 --- a/KubernetesWorkflow/ResourceLimits.cs +++ b/KubernetesWorkflow/ContainerResources.cs @@ -2,25 +2,36 @@ namespace KubernetesWorkflow { - public class ResourceLimits + public class ContainerResources { - public ResourceLimits(int milliCPUs, ByteSize memory) + public ContainerResourceSet Requests { get; set; } = new ContainerResourceSet(); + public ContainerResourceSet Limits { get; set; } = new ContainerResourceSet(); + + public override string ToString() + { + return $"requests:{Requests}, limits:{Limits}"; + } + } + + public class ContainerResourceSet + { + public ContainerResourceSet(int milliCPUs, ByteSize memory) { MilliCPUs = milliCPUs; Memory = memory; } - public ResourceLimits(int milliCPUs) + public ContainerResourceSet(int milliCPUs) : this(milliCPUs, new ByteSize(0)) { } - public ResourceLimits(ByteSize memory) + public ContainerResourceSet(ByteSize memory) : this(0, memory) { } - public ResourceLimits() + public ContainerResourceSet() : this(0) { } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 8c35c59..cfb4315 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -432,20 +432,21 @@ namespace KubernetesWorkflow { return new V1ResourceRequirements { - Limits = CreateResourceLimit(recipe.ResourceLimits) + Requests = CreateResourceQuantities(recipe.Resources.Requests), + Limits = CreateResourceQuantities(recipe.Resources.Limits) }; } - private Dictionary CreateResourceLimit(ResourceLimits limits) + private Dictionary CreateResourceQuantities(ContainerResourceSet set) { var result = new Dictionary(); - if (limits.MilliCPUs != 0) + if (set.MilliCPUs != 0) { - result.Add("cpu", new ResourceQuantity($"{limits.MilliCPUs}m")); + result.Add("cpu", new ResourceQuantity($"{set.MilliCPUs}m")); } - if (limits.Memory.SizeInBytes != 0) + if (set.Memory.SizeInBytes != 0) { - result.Add("memory", new ResourceQuantity(limits.Memory.ToSuffixNotation())); + result.Add("memory", new ResourceQuantity(set.Memory.ToSuffixNotation())); } return result; } From d074f64c422b66819e9566c7cbefabc7d0db12b4 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 8 Sep 2023 14:03:47 +0200 Subject: [PATCH 09/12] a --- .../DownloadConnectivityTests/FullyConnectedDownloadTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index 02ad9b0..9fb5630 100644 --- a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -26,8 +26,8 @@ namespace Tests.DownloadConnectivityTests [Test] [Combinatorial] public void FullyConnectedDownloadTest( - [Values(1, 3, 5)] int numberOfNodes, - [Values(1, 10)] int sizeMBs) + [Values(3, 5)] int numberOfNodes, + [Values(10, 80)] int sizeMBs) { SetupCodexNodes(numberOfNodes); From a5749824fbbd6fb97f3493fcf91dfb8d47edd52f Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 8 Sep 2023 15:43:10 +0200 Subject: [PATCH 10/12] Update to full connectivity helper --- .../Helpers/FullConnectivityHelper.cs | 32 ++++--------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/DistTestCore/Helpers/FullConnectivityHelper.cs b/DistTestCore/Helpers/FullConnectivityHelper.cs index 68dfdbb..b69af0c 100644 --- a/DistTestCore/Helpers/FullConnectivityHelper.cs +++ b/DistTestCore/Helpers/FullConnectivityHelper.cs @@ -1,7 +1,6 @@ using DistTestCore.Codex; using Logging; using NUnit.Framework; -using Utils; namespace DistTestCore.Helpers { @@ -35,10 +34,9 @@ namespace DistTestCore.Helpers var entries = CreateEntries(nodes); var pairs = CreatePairs(entries); - RetryWhilePairs(pairs, () => - { - CheckAndRemoveSuccessful(pairs); - }); + // Each pair gets two chances. + CheckAndRemoveSuccessful(pairs); + CheckAndRemoveSuccessful(pairs); if (pairs.Any()) { @@ -54,35 +52,19 @@ namespace DistTestCore.Helpers } } - private static void RetryWhilePairs(List pairs, Action action) - { - var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2); - while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow) - { - action(); - - Time.Sleep(TimeSpan.FromSeconds(2)); - } - } - private void CheckAndRemoveSuccessful(List pairs) { - // For large sets, don't try and do all of them at once. - var selectedPair = pairs.Take(20).ToArray(); - var pairDetails = new List(); - - foreach (var pair in selectedPair) + var results = new List(); + foreach (var pair in pairs.ToArray()) { pair.Check(); - if (pair.Success) { - pairDetails.AddRange(pair.GetResultMessages()); + results.AddRange(pair.GetResultMessages()); pairs.Remove(pair); } } - - Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}"); + Log($"Connections successful:{Nl}{string.Join(Nl, results)}"); } private Entry[] CreateEntries(CodexAccess[] nodes) From 1b3b45a6a4b868ca97822d723a80b68992c1a6ff Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 8 Sep 2023 18:13:54 +0200 Subject: [PATCH 11/12] Bumps codex resources --- DistTestCore/Codex/CodexContainerRecipe.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index c9c0096..c928b3e 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -22,8 +22,8 @@ namespace DistTestCore.Codex { Image = GetDockerImage(); - Resources.Requests = new ContainerResourceSet(milliCPUs: 250, memory: 512.MB()); - Resources.Limits = new ContainerResourceSet(milliCPUs: 1000, memory: 2.GB()); + Resources.Requests = new ContainerResourceSet(milliCPUs: 1000, memory: 6.GB()); + Resources.Limits = new ContainerResourceSet(milliCPUs: 4000, memory: 12.GB()); } protected override void InitializeRecipe(StartupConfig startupConfig) From 0bcca1e8459baddd0b4630212d8e45433c2fb661 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 11 Sep 2023 09:58:23 +0200 Subject: [PATCH 12/12] unused usings --- KubernetesWorkflow/K8sController.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index cfb4315..c162bcf 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -1,9 +1,6 @@ using k8s; using k8s.Models; using Logging; -using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources; -using System.Collections.Generic; -using System.Xml.Linq; using Utils; namespace KubernetesWorkflow