From 7de0e5a1c4ad2a0e0f1d6a247ef1bd492d24f2d7 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 14 Nov 2023 10:16:00 +0100 Subject: [PATCH] Sets up tests-runners as avoided scheduling affinity. --- Framework/KubernetesWorkflow/K8sController.cs | 37 +++++++++++++++++++ .../Recipe/ContainerRecipe.cs | 5 ++- .../Recipe/ContainerRecipeFactory.cs | 9 ++++- .../Recipe/SchedulingAffinity.cs | 18 +++++++++ .../CodexContractsContainerRecipe.cs | 2 + .../DiscordBotContainerRecipe.cs | 2 + .../CodexPlugin/CodexContainerRecipe.cs | 2 + .../GethPlugin/GethContainerRecipe.cs | 2 + .../PrometheusContainerRecipe.cs | 2 + 9 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 Framework/KubernetesWorkflow/Recipe/SchedulingAffinity.cs diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 4d4ca11..2251336 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -335,6 +335,7 @@ namespace KubernetesWorkflow }, Spec = new V1PodSpec { + Affinity = CreatePodAffinity(containerRecipes), NodeSelector = CreateNodeSelector(location), Containers = CreateDeploymentContainers(containerRecipes), Volumes = CreateVolumes(containerRecipes) @@ -367,6 +368,42 @@ namespace KubernetesWorkflow }; } + private V1Affinity? CreatePodAffinity(ContainerRecipe[] recipes) + { + var notIns = recipes + .Select(r => r.SchedulingAffinity.NotIn) + .Where(n => !string.IsNullOrEmpty(n)) + .Distinct() + .ToList(); + + if (!notIns.Any()) return null; + + return new V1Affinity + { + NodeAffinity = new V1NodeAffinity + { + RequiredDuringSchedulingIgnoredDuringExecution = new V1NodeSelector + { + NodeSelectorTerms = new List + { + new V1NodeSelectorTerm + { + MatchExpressions = new List + { + new V1NodeSelectorRequirement + { + Key = "workload-type", + OperatorProperty = "NotIn", + Values = notIns + } + } + } + } + } + } + }; + } + private K8sNodeLabel? GetNodeLabelForLocation(ILocation location) { var l = (Location)location; diff --git a/Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs b/Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs index b66bd42..1865dc7 100644 --- a/Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs +++ b/Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs @@ -2,12 +2,13 @@ { public class ContainerRecipe { - public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals) + public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals) { Number = number; NameOverride = nameOverride; Image = image; Resources = resources; + SchedulingAffinity = schedulingAffinity; ExposedPorts = exposedPorts; InternalPorts = internalPorts; EnvVars = envVars; @@ -32,6 +33,7 @@ public int Number { get; } public string? NameOverride { get; } public ContainerResources Resources { get; } + public SchedulingAffinity SchedulingAffinity { get; } public string Image { get; } public Port[] ExposedPorts { get; } public Port[] InternalPorts { get; } @@ -53,6 +55,7 @@ $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " + $"envVars: {string.Join(",", EnvVars.Select(v => v.ToString()))}, " + $"limits: {Resources}, " + + $"affinity: {SchedulingAffinity}, " + $"volumes: {string.Join(",", Volumes.Select(v => $"'{v.MountPath}'"))}"; } } diff --git a/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs b/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs index 134b280..3b2b941 100644 --- a/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs +++ b/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs @@ -13,6 +13,7 @@ namespace KubernetesWorkflow.Recipe private readonly List additionals = new List(); private RecipeComponentFactory factory = null!; private ContainerResources resources = new ContainerResources(); + private SchedulingAffinity schedulingAffinity = new SchedulingAffinity(); public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config) { @@ -22,7 +23,7 @@ namespace KubernetesWorkflow.Recipe Initialize(config); - var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, + var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray(), @@ -40,6 +41,7 @@ namespace KubernetesWorkflow.Recipe additionals.Clear(); this.factory = null!; resources = new ContainerResources(); + schedulingAffinity = new SchedulingAffinity(); return recipe; } @@ -121,6 +123,11 @@ namespace KubernetesWorkflow.Recipe SetResourcesRequest(new ContainerResourceSet(milliCPUs, memory)); } + protected void SetSchedulingAffinity(string notIn) + { + schedulingAffinity = new SchedulingAffinity(notIn); + } + // Disabled following a possible bug in the k8s cluster that will throttle containers much more than is // called for if they have resource limits defined. //protected void SetResourceLimits(int milliCPUs, ByteSize memory) diff --git a/Framework/KubernetesWorkflow/Recipe/SchedulingAffinity.cs b/Framework/KubernetesWorkflow/Recipe/SchedulingAffinity.cs new file mode 100644 index 0000000..8bc4c84 --- /dev/null +++ b/Framework/KubernetesWorkflow/Recipe/SchedulingAffinity.cs @@ -0,0 +1,18 @@ +namespace KubernetesWorkflow.Recipe +{ + public class SchedulingAffinity + { + public SchedulingAffinity(string? notIn = null) + { + NotIn = notIn; + } + + public string? NotIn { get; } + + public override string ToString() + { + if (string.IsNullOrEmpty(NotIn)) return "none"; + return "notIn:" + NotIn; + } + } +} diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs index d6dd92f..3e5faa1 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs @@ -21,6 +21,8 @@ namespace CodexContractsPlugin var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag); + SetSchedulingAffinity(notIn: "tests-runners"); + AddEnvVar("DISTTEST_NETWORK_URL", address.ToString()); AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork"); AddEnvVar("KEEP_ALIVE", "1"); diff --git a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs index b4b1432..e1596b4 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs @@ -13,6 +13,8 @@ namespace CodexDiscordBotPlugin { var config = startupConfig.Get(); + SetSchedulingAffinity(notIn: "tests-runners"); + AddEnvVar("TOKEN", config.Token); AddEnvVar("SERVERNAME", config.ServerName); AddEnvVar("ADMINROLE", config.AdminRoleName); diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index bfdf52b..744ae37 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -29,6 +29,8 @@ namespace CodexPlugin SetResourcesRequest(milliCPUs: 100, memory: 100.MB()); //SetResourceLimits(milliCPUs: 4000, memory: 12.GB()); + SetSchedulingAffinity(notIn: "tests-runners"); + var config = startupConfig.Get(); var apiPort = CreateApiPort(config, ApiPortTag); diff --git a/ProjectPlugins/GethPlugin/GethContainerRecipe.cs b/ProjectPlugins/GethPlugin/GethContainerRecipe.cs index f9eaf79..35b994c 100644 --- a/ProjectPlugins/GethPlugin/GethContainerRecipe.cs +++ b/ProjectPlugins/GethPlugin/GethContainerRecipe.cs @@ -24,6 +24,8 @@ namespace GethPlugin var args = CreateArgs(config); + SetSchedulingAffinity(notIn: "tests-runners"); + AddEnvVar("GETH_ARGS", args); } diff --git a/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs b/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs index a4dfdb4..e132a8f 100644 --- a/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs +++ b/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs @@ -14,6 +14,8 @@ namespace MetricsPlugin { var config = startupConfig.Get(); + SetSchedulingAffinity(notIn: "tests-runners"); + AddExposedPortAndVar("PROM_PORT", PortTag); AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64); }