Merge branch 'feature/scheduling-affinity'

Framework/KubernetesWorkflow/K8sController.cs

@@ -335,6 +335,7 @@ namespace KubernetesWorkflow
                         },
                         Spec = new V1PodSpec
                         {
+                            Affinity = CreatePodAffinity(containerRecipes),
                             NodeSelector = CreateNodeSelector(location),
                             Containers = CreateDeploymentContainers(containerRecipes),
                             Volumes = CreateVolumes(containerRecipes)
@@ -367,6 +368,42 @@ namespace KubernetesWorkflow
             };
         }
 
+        private V1Affinity? CreatePodAffinity(ContainerRecipe[] recipes)
+        {
+            var notIns = recipes
+                .Select(r => r.SchedulingAffinity.NotIn)
+                .Where(n => !string.IsNullOrEmpty(n))
+                .Distinct()
+                .ToList();
+
+            if (!notIns.Any()) return null;
+
+            return new V1Affinity
+            {
+                NodeAffinity = new V1NodeAffinity
+                {
+                    RequiredDuringSchedulingIgnoredDuringExecution = new V1NodeSelector
+                    {
+                        NodeSelectorTerms = new List<V1NodeSelectorTerm>
+                        { 
+                            new V1NodeSelectorTerm
+                            {
+                                MatchExpressions = new List<V1NodeSelectorRequirement>
+                                {
+                                    new V1NodeSelectorRequirement
+                                    {
+                                        Key = "workload-type",
+                                        OperatorProperty = "NotIn",
+                                        Values = notIns
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            };
+        }
+
         private K8sNodeLabel? GetNodeLabelForLocation(ILocation location)
         {
             var l = (Location)location;

Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs

@@ -2,12 +2,13 @@
 {
     public class ContainerRecipe
     {
-        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
+        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
         {
             Number = number;
             NameOverride = nameOverride;
             Image = image;
             Resources = resources;
+            SchedulingAffinity = schedulingAffinity;
             ExposedPorts = exposedPorts;
             InternalPorts = internalPorts;
             EnvVars = envVars;
@@ -32,6 +33,7 @@
         public int Number { get; }
         public string? NameOverride { get; }
         public ContainerResources Resources { get; }
+        public SchedulingAffinity SchedulingAffinity { get; }
         public string Image { get; }
         public Port[] ExposedPorts { get; }
         public Port[] InternalPorts { get; }
@@ -53,6 +55,7 @@
                 $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
                 $"envVars: {string.Join(",", EnvVars.Select(v => v.ToString()))}, " +
                 $"limits: {Resources}, " +
+                $"affinity: {SchedulingAffinity}, " +
                 $"volumes: {string.Join(",", Volumes.Select(v => $"'{v.MountPath}'"))}";
         }
     }

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -13,6 +13,7 @@ namespace KubernetesWorkflow.Recipe
         private readonly List<object> additionals = new List<object>();
         private RecipeComponentFactory factory = null!;
         private ContainerResources resources = new ContainerResources();
+        private SchedulingAffinity schedulingAffinity = new SchedulingAffinity();
 
         public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config)
         {
@@ -22,7 +23,7 @@ namespace KubernetesWorkflow.Recipe
 
             Initialize(config);
 
-            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources,
+            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity,
                 exposedPorts.ToArray(),
                 internalPorts.ToArray(),
                 envVars.ToArray(),
@@ -40,6 +41,7 @@ namespace KubernetesWorkflow.Recipe
             additionals.Clear();
             this.factory = null!;
             resources = new ContainerResources();
+            schedulingAffinity = new SchedulingAffinity();
 
             return recipe;
         }
@@ -121,6 +123,11 @@ namespace KubernetesWorkflow.Recipe
             SetResourcesRequest(new ContainerResourceSet(milliCPUs, memory));
         }
 
+        protected void SetSchedulingAffinity(string notIn)
+        {
+            schedulingAffinity = new SchedulingAffinity(notIn);
+        }
+
         // Disabled following a possible bug in the k8s cluster that will throttle containers much more than is
         // called for if they have resource limits defined.
         //protected void SetResourceLimits(int milliCPUs, ByteSize memory)

Framework/KubernetesWorkflow/Recipe/SchedulingAffinity.cs

@@ -0,0 +1,18 @@
+namespace KubernetesWorkflow.Recipe
+{
+    public class SchedulingAffinity
+    {
+        public SchedulingAffinity(string? notIn = null)
+        {
+            NotIn = notIn;
+        }
+
+        public string? NotIn { get; }
+
+        public override string ToString()
+        {
+            if (string.IsNullOrEmpty(NotIn)) return "none";
+            return "notIn:" + NotIn;
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -21,6 +21,8 @@ namespace CodexContractsPlugin
 
             var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
 
+            SetSchedulingAffinity(notIn: "tests-runners");
+
             AddEnvVar("DISTTEST_NETWORK_URL", address.ToString());
             AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork");
             AddEnvVar("KEEP_ALIVE", "1");

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs

@@ -13,6 +13,8 @@ namespace CodexDiscordBotPlugin
         {
             var config = startupConfig.Get<DiscordBotStartupConfig>();
 
+            SetSchedulingAffinity(notIn: "tests-runners");
+
             AddEnvVar("TOKEN", config.Token);
             AddEnvVar("SERVERNAME", config.ServerName);
             AddEnvVar("ADMINROLE", config.AdminRoleName);

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -29,6 +29,8 @@ namespace CodexPlugin
             SetResourcesRequest(milliCPUs: 100, memory: 100.MB());
             //SetResourceLimits(milliCPUs: 4000, memory: 12.GB());
 
+            SetSchedulingAffinity(notIn: "tests-runners");
+
             var config = startupConfig.Get<CodexStartupConfig>();
 
             var apiPort = CreateApiPort(config, ApiPortTag);

ProjectPlugins/GethPlugin/GethContainerRecipe.cs

@@ -24,6 +24,8 @@ namespace GethPlugin
 
             var args = CreateArgs(config);
 
+            SetSchedulingAffinity(notIn: "tests-runners");
+
             AddEnvVar("GETH_ARGS", args);
         }
 

ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs

@@ -14,6 +14,8 @@ namespace MetricsPlugin
         {
             var config = startupConfig.Get<PrometheusStartupConfig>();
 
+            SetSchedulingAffinity(notIn: "tests-runners");
+
             AddExposedPortAndVar("PROM_PORT", PortTag);
             AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
         }