Automatically map location enum to available k8s nodes.

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -49,7 +49,7 @@ namespace DistTestCore.Codex
                 var companionNodeAccount = companionNode.Accounts[Index];
                 Additional(companionNodeAccount);
 
-                var ip = companionNode.RunningContainer.Pod.Ip;
+                var ip = companionNode.RunningContainer.Pod.PodInfo.Ip;
                 var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number;
 
                 AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}");

DistTestCore/CodexStarter.cs

@@ -28,7 +28,8 @@ namespace DistTestCore
             var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
 
             var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
-            LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: {group.Describe()}");
+            var podInfo = group.Containers.RunningPod.PodInfo;
+            LogEnd($"Started {codexSetup.NumberOfNodes} nodes at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}");
             LogSeparator();
             return group;
         }

DistTestCore/Configuration.cs

@@ -28,12 +28,7 @@ namespace DistTestCore
                 k8sNamespacePrefix: "ct-",
                 kubeConfigFile: kubeConfigFile,
                 operationTimeout: timeSet.K8sOperationTimeout(),
-                retryDelay: timeSet.WaitForK8sServiceDelay(),
+                retryDelay: timeSet.WaitForK8sServiceDelay()
-                locationMap: new[]
-                {
-                    new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),
-                    new ConfigurationLocationEntry(Location.BensLaptop, "worker02"),
-                } 
             );
         }
 

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -138,7 +138,7 @@ namespace DistTestCore.Helpers
                 if (peer == null) return $"peerId: {node.peerId} is not known.";
 
                 var n = (OnlineCodexNode)peer.Node;
-                var ip = n.CodexAccess.Container.Pod.Ip;
+                var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
                 var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
                 return $"{ip}:{discPort.Number}";
             }

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace
         private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer)
         {
             var startupConfig = new StartupConfig();
-            var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
+            var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.PodInfo.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
             startupConfig.Add(contractsConfig);
             return startupConfig;
         }

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -58,7 +58,7 @@ namespace DistTestCore.Marketplace
             var httpPort = AddExposedPort(tag: HttpPortTag);
 
             var bootPubKey = config.BootstrapNode.PubKey;
-            var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip;
+            var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.PodInfo.Ip;
             var bootPort = config.BootstrapNode.DiscoveryPort.Number;
             var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}";
 

DistTestCore/Metrics/MetricsQuery.cs

@@ -119,7 +119,7 @@ namespace DistTestCore.Metrics
 
         private string GetInstanceNameForNode(RunningContainer node)
         {
-            var ip = node.Pod.Ip;
+            var ip = node.Pod.PodInfo.Ip;
             var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
             return $"{ip}:{port}";
         }

DistTestCore/OnlineCodexNode.cs

@@ -124,7 +124,7 @@ namespace DistTestCore
 
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.Ip);
+            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip);
         }
 
         private void DownloadToFile(string contentId, TestFile file)

DistTestCore/PrometheusStarter.cs

@@ -44,7 +44,7 @@ namespace DistTestCore
 
             foreach (var node in nodes)
             {
-                var ip = node.Pod.Ip;
+                var ip = node.Pod.PodInfo.Ip;
                 var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
                 config += $"          - '{ip}:{port}'\n";
             }

KubernetesWorkflow/CommandRunner.cs

@@ -28,7 +28,7 @@ namespace KubernetesWorkflow
             var input = new[] { command }.Concat(arguments).ToArray();
 
             Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
-                pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
+                pod.PodInfo.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
         }
 
         public string GetStdOut()

KubernetesWorkflow/Configuration.cs

@@ -2,31 +2,17 @@
 {
     public class Configuration
     {
-        public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
+        public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay)
         {
             K8sNamespacePrefix = k8sNamespacePrefix;
             KubeConfigFile = kubeConfigFile;
             OperationTimeout = operationTimeout;
             RetryDelay = retryDelay;
-            LocationMap = locationMap;
         }
 
         public string K8sNamespacePrefix { get; }
         public string? KubeConfigFile { get; }
         public TimeSpan OperationTimeout { get; }
         public TimeSpan RetryDelay { get; }
-        public ConfigurationLocationEntry[] LocationMap { get; }
-    }
-
-    public class ConfigurationLocationEntry
-    {
-        public ConfigurationLocationEntry(Location location, string workerName)
-        {
-            Location = location;
-            WorkerName = workerName;
-        }
-
-        public Location Location { get; }
-        public string WorkerName { get; }
     }
 }

KubernetesWorkflow/K8sCluster.cs

@@ -11,6 +11,7 @@ namespace KubernetesWorkflow
        
         public Configuration Configuration { get; }
         public string HostAddress { get; private set; } = string.Empty;
+        public string[] AvailableK8sNodes { get; set; } = new string[0];
 
         public KubernetesClientConfiguration GetK8sClientConfig()
         {
@@ -21,8 +22,16 @@ namespace KubernetesWorkflow
 
         public string GetNodeLabelForLocation(Location location)
         {
-            if (location == Location.Unspecified) return string.Empty;
+            switch (location)
-            return Configuration.LocationMap.Single(l => l.Location == location).WorkerName;
+            {
+                case Location.One:
+                    return K8sNodeIfAvailable(0);
+                case Location.Two:
+                    return K8sNodeIfAvailable(1);
+                case Location.Three:
+                    return K8sNodeIfAvailable(2);
+            }
+            return string.Empty;
         }
 
         public TimeSpan K8sOperationTimeout()
@@ -59,5 +68,11 @@ namespace KubernetesWorkflow
                 HostAddress = config.Host;
             }
         }
+
+        private string K8sNodeIfAvailable(int index)
+        {
+            if (AvailableK8sNodes.Length <= index) return string.Empty;
+            return AvailableK8sNodes[index];
+        }
     }
 }

KubernetesWorkflow/K8sController.cs

@@ -32,13 +32,14 @@ namespace KubernetesWorkflow
         public RunningPod BringOnline(ContainerRecipe[] containerRecipes, Location location)
         {
             log.Debug();
+            DiscoverK8sNodes();
             EnsureTestNamespace();
 
             var deploymentName = CreateDeployment(containerRecipes, location);
             var (serviceName, servicePortsMap) = CreateService(containerRecipes);
-            var (podName, podIp) = FetchNewPod();
+            var podInfo = FetchNewPod();
 
-            return new RunningPod(cluster, podName, podIp, deploymentName, serviceName, servicePortsMap);
+            return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap);
         }
 
         public void Stop(RunningPod pod)
@@ -47,13 +48,13 @@ namespace KubernetesWorkflow
             if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName);
             DeleteDeployment(pod.DeploymentName);
             WaitUntilDeploymentOffline(pod.DeploymentName);
-            WaitUntilPodOffline(pod.Name);
+            WaitUntilPodOffline(pod.PodInfo.Name);
         }
 
         public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
         {
             log.Debug();
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name));
             logHandler.Log(stream);
         }
 
@@ -106,6 +107,28 @@ namespace KubernetesWorkflow
             }
         }
 
+        #region Discover K8s Nodes
+
+        private void DiscoverK8sNodes()
+        {
+            if (cluster.AvailableK8sNodes == null || !cluster.AvailableK8sNodes.Any())
+            {
+                cluster.AvailableK8sNodes = GetAvailableK8sNodes();
+                if (cluster.AvailableK8sNodes.Length < 3)
+                {
+                    log.Debug($"Warning: For full location support, at least 3 Kubernetes Nodes are required in the cluster. Nodes found: '{string.Join(",", cluster.AvailableK8sNodes)}'.");
+                }
+            }
+        }
+
+        private string[] GetAvailableK8sNodes()
+        {
+            var nodes = client.Run(c => c.ListNode());
+            return nodes.Items.Select(i => i.Metadata.Name).ToArray();
+        }
+
+        #endregion
+
         #region Namespace management
 
         private string K8sTestNamespace { get; }
@@ -537,7 +560,7 @@ namespace KubernetesWorkflow
 
         #endregion
 
-        private (string, string) FetchNewPod()
+        private PodInfo FetchNewPod()
         {
             var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
 
@@ -547,12 +570,13 @@ namespace KubernetesWorkflow
             var newPod = newPods.Single();
             var name = newPod.Name();
             var ip = newPod.Status.PodIP;
+            var k8sNodeName = newPod.Spec.NodeName;
 
             if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure.");
             if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure.");
 
             knownPods.Add(name);
-            return (name, ip);
+            return new PodInfo(name, ip, k8sNodeName);
         }
     }
 }

KubernetesWorkflow/Location.cs

@@ -3,7 +3,8 @@
     public enum Location
     {
         Unspecified,
-        BensLaptop,
+        One,
-        BensOldGamingMachine
+        Two,
+        Three,
     }
 }

KubernetesWorkflow/RunningPod.cs

@@ -4,19 +4,17 @@
     {
         private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap;
 
-        public RunningPod(K8sCluster cluster, string name, string ip, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
+        public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
         {
             Cluster = cluster;
-            Name = name;
+            PodInfo = podInfo;
-            Ip = ip;
             DeploymentName = deploymentName;
             ServiceName = serviceName;
             this.servicePortMap = servicePortMap;
         }
 
         public K8sCluster Cluster { get; }
-        public string Name { get; }
+        public PodInfo PodInfo { get; }
-        public string Ip { get; }
         internal string DeploymentName { get; }
         internal string ServiceName { get; }
 
@@ -25,4 +23,18 @@
             return servicePortMap[containerRecipe];
         }
     }
+
+    public class PodInfo
+    {
+        public PodInfo(string podName, string podIp, string k8sNodeName)
+        {
+            Name = podName;
+            Ip = podIp;
+            K8SNodeName = k8sNodeName;
+        }
+
+        public string Name { get; }
+        public string Ip { get; }
+        public string K8SNodeName { get; }
+    }
 }

Tests/BasicTests/TwoClientTests.cs

@@ -28,11 +28,10 @@ namespace Tests.BasicTests
         }
 
         [Test]
-        [Ignore("Requires Location map to be configured for k8s cluster.")]
         public void TwoClientsTwoLocationsTest()
         {
-            var primary = SetupCodexNode(s => s.At(Location.BensLaptop));
+            var primary = SetupCodexNode(s => s.At(Location.One));
-            var secondary = SetupCodexNode(s => s.At(Location.BensOldGamingMachine));
+            var secondary = SetupCodexNode(s => s.At(Location.Two));
 
             PerformTwoClientTest(primary, secondary);
         }