From 3761e236a330453000b69fbd65e7db70244d0f16 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 23 Nov 2023 14:50:54 +0100 Subject: [PATCH] Sets node critical priority for codex and geth nodes --- Framework/KubernetesWorkflow/K8sController.cs | 10 ++++++++++ Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs | 4 +++- .../Recipe/ContainerRecipeFactory.cs | 9 ++++++++- ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs | 1 + ProjectPlugins/GethPlugin/GethContainerRecipe.cs | 1 + 5 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index 9cde3fb..e8c28ef 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -335,6 +335,7 @@ namespace KubernetesWorkflow }, Spec = new V1PodSpec { + PriorityClassName = GetPriorityClassName(containerRecipes), Affinity = CreatePodAffinity(containerRecipes), NodeSelector = CreateNodeSelector(location), Containers = CreateDeploymentContainers(containerRecipes), @@ -410,6 +411,15 @@ namespace KubernetesWorkflow return l.NodeLabel; } + private string GetPriorityClassName(ContainerRecipe[] containerRecipes) + { + if (containerRecipes.Any(c => c.SetCriticalPriority)) + { + return "system-node-critical"; + } + return null!; + } + private IDictionary GetSelector(ContainerRecipe[] containerRecipes) { return containerRecipes.First().PodLabels.GetLabels(); diff --git a/Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs b/Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs index 1865dc7..fb7c8a4 100644 --- a/Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs +++ b/Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs @@ -2,13 +2,14 @@ { public class ContainerRecipe { - public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals) + public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals) { Number = number; NameOverride = nameOverride; Image = image; Resources = resources; SchedulingAffinity = schedulingAffinity; + SetCriticalPriority = setCriticalPriority; ExposedPorts = exposedPorts; InternalPorts = internalPorts; EnvVars = envVars; @@ -34,6 +35,7 @@ public string? NameOverride { get; } public ContainerResources Resources { get; } public SchedulingAffinity SchedulingAffinity { get; } + public bool SetCriticalPriority { get; } public string Image { get; } public Port[] ExposedPorts { get; } public Port[] InternalPorts { get; } diff --git a/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs b/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs index 3b2b941..c2ba8d8 100644 --- a/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs +++ b/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs @@ -14,6 +14,7 @@ namespace KubernetesWorkflow.Recipe private RecipeComponentFactory factory = null!; private ContainerResources resources = new ContainerResources(); private SchedulingAffinity schedulingAffinity = new SchedulingAffinity(); + private bool setCriticalPriority; public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config) { @@ -23,7 +24,7 @@ namespace KubernetesWorkflow.Recipe Initialize(config); - var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, + var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, setCriticalPriority, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray(), @@ -42,6 +43,7 @@ namespace KubernetesWorkflow.Recipe this.factory = null!; resources = new ContainerResources(); schedulingAffinity = new SchedulingAffinity(); + setCriticalPriority = false; return recipe; } @@ -128,6 +130,11 @@ namespace KubernetesWorkflow.Recipe schedulingAffinity = new SchedulingAffinity(notIn); } + protected void SetSystemCriticalPriority() + { + setCriticalPriority = true; + } + // Disabled following a possible bug in the k8s cluster that will throttle containers much more than is // called for if they have resource limits defined. //protected void SetResourceLimits(int milliCPUs, ByteSize memory) diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 744ae37..5298279 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -30,6 +30,7 @@ namespace CodexPlugin //SetResourceLimits(milliCPUs: 4000, memory: 12.GB()); SetSchedulingAffinity(notIn: "tests-runners"); + SetSystemCriticalPriority(); var config = startupConfig.Get(); diff --git a/ProjectPlugins/GethPlugin/GethContainerRecipe.cs b/ProjectPlugins/GethPlugin/GethContainerRecipe.cs index 35b994c..9db6876 100644 --- a/ProjectPlugins/GethPlugin/GethContainerRecipe.cs +++ b/ProjectPlugins/GethPlugin/GethContainerRecipe.cs @@ -25,6 +25,7 @@ namespace GethPlugin var args = CreateArgs(config); SetSchedulingAffinity(notIn: "tests-runners"); + SetSystemCriticalPriority(); AddEnvVar("GETH_ARGS", args); }