Sets node critical priority for codex and geth nodes

Framework/KubernetesWorkflow/K8sController.cs

@@ -335,6 +335,7 @@ namespace KubernetesWorkflow
                         },
                         Spec = new V1PodSpec
                         {
+                            PriorityClassName = GetPriorityClassName(containerRecipes),
                             Affinity = CreatePodAffinity(containerRecipes),
                             NodeSelector = CreateNodeSelector(location),
                             Containers = CreateDeploymentContainers(containerRecipes),
@@ -410,6 +411,15 @@ namespace KubernetesWorkflow
             return l.NodeLabel;
         }
 
+        private string GetPriorityClassName(ContainerRecipe[] containerRecipes)
+        {
+            if (containerRecipes.Any(c => c.SetCriticalPriority))
+            {
+                return "system-node-critical";
+            }
+            return null!;
+        }
+
         private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes)
         {
             return containerRecipes.First().PodLabels.GetLabels();

Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs

@@ -2,13 +2,14 @@
 {
     public class ContainerRecipe
     {
-        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
+        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
         {
             Number = number;
             NameOverride = nameOverride;
             Image = image;
             Resources = resources;
             SchedulingAffinity = schedulingAffinity;
+            SetCriticalPriority = setCriticalPriority;
             ExposedPorts = exposedPorts;
             InternalPorts = internalPorts;
             EnvVars = envVars;
@@ -34,6 +35,7 @@
         public string? NameOverride { get; }
         public ContainerResources Resources { get; }
         public SchedulingAffinity SchedulingAffinity { get; }
+        public bool SetCriticalPriority { get; }
         public string Image { get; }
         public Port[] ExposedPorts { get; }
         public Port[] InternalPorts { get; }

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -14,6 +14,7 @@ namespace KubernetesWorkflow.Recipe
         private RecipeComponentFactory factory = null!;
         private ContainerResources resources = new ContainerResources();
         private SchedulingAffinity schedulingAffinity = new SchedulingAffinity();
+        private bool setCriticalPriority;
 
         public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config)
         {
@@ -23,7 +24,7 @@ namespace KubernetesWorkflow.Recipe
 
             Initialize(config);
 
-            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity,
+            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, setCriticalPriority,
                 exposedPorts.ToArray(),
                 internalPorts.ToArray(),
                 envVars.ToArray(),
@@ -42,6 +43,7 @@ namespace KubernetesWorkflow.Recipe
             this.factory = null!;
             resources = new ContainerResources();
             schedulingAffinity = new SchedulingAffinity();
+            setCriticalPriority = false;
 
             return recipe;
         }
@@ -128,6 +130,11 @@ namespace KubernetesWorkflow.Recipe
             schedulingAffinity = new SchedulingAffinity(notIn);
         }
 
+        protected void SetSystemCriticalPriority()
+        {
+            setCriticalPriority = true;
+        }
+
         // Disabled following a possible bug in the k8s cluster that will throttle containers much more than is
         // called for if they have resource limits defined.
         //protected void SetResourceLimits(int milliCPUs, ByteSize memory)

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -30,6 +30,7 @@ namespace CodexPlugin
             //SetResourceLimits(milliCPUs: 4000, memory: 12.GB());
 
             SetSchedulingAffinity(notIn: "tests-runners");
+            SetSystemCriticalPriority();
 
             var config = startupConfig.Get<CodexStartupConfig>();
 

ProjectPlugins/GethPlugin/GethContainerRecipe.cs

@@ -25,6 +25,7 @@ namespace GethPlugin
             var args = CreateArgs(config);
 
             SetSchedulingAffinity(notIn: "tests-runners");
+            SetSystemCriticalPriority();
 
             AddEnvVar("GETH_ARGS", args);
         }