Merge branch 'feature/configurability'

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -49,7 +49,7 @@ namespace DistTestCore.Codex
                 var companionNodeAccount = companionNode.Accounts[Index];
                 Additional(companionNodeAccount);
 
-                var ip = companionNode.RunningContainer.Pod.Ip;
+                var ip = companionNode.RunningContainer.Pod.PodInfo.Ip;
                 var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number;
 
                 AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}");

DistTestCore/CodexStarter.cs

@@ -28,7 +28,8 @@ namespace DistTestCore
             var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
 
             var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
-            LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: {group.Describe()}");
+            var podInfo = group.Containers.RunningPod.PodInfo;
+            LogEnd($"Started {codexSetup.NumberOfNodes} nodes at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}");
             LogSeparator();
             return group;
         }

DistTestCore/Configuration.cs

@@ -5,39 +5,51 @@ namespace DistTestCore
 {
     public class Configuration
     {
+        private readonly string? kubeConfigFile;
+        private readonly string logPath;
+        private readonly bool logDebug;
+        private readonly string dataFilesPath;
+        private readonly CodexLogLevel codexLogLevel;
+        private readonly TestRunnerLocation runnerLocation;
+
+        public Configuration()
+        {
+            kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
+            logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
+            logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
+            dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
+            codexLogLevel = ParseEnum<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
+            runnerLocation = ParseEnum<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
+        }
+
         public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
         {
             return new KubernetesWorkflow.Configuration(
                 k8sNamespacePrefix: "ct-",
-                kubeConfigFile: null,
+                kubeConfigFile: kubeConfigFile,
                 operationTimeout: timeSet.K8sOperationTimeout(),
-                retryDelay: timeSet.WaitForK8sServiceDelay(),
+                retryDelay: timeSet.WaitForK8sServiceDelay()
-                locationMap: new[]
-                {
-                    new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),
-                    new ConfigurationLocationEntry(Location.BensLaptop, "worker02"),
-                } 
             );
         }
 
         public Logging.LogConfig GetLogConfig()
         {
-            return new Logging.LogConfig("CodexTestLogs", debugEnabled: false);
+            return new Logging.LogConfig(logPath, debugEnabled: logDebug);
         }
 
         public string GetFileManagerFolder()
         {
-            return "TestDataFiles";
+            return dataFilesPath;
         }
 
         public CodexLogLevel GetCodexLogLevel()
         {
-            return CodexLogLevel.Trace;
+            return codexLogLevel;
         }
 
         public TestRunnerLocation GetTestRunnerLocation()
         {
-            return TestRunnerLocation.ExternalToCluster;
+            return runnerLocation;
         }
 
         public RunningContainerAddress GetAddress(RunningContainer container)
@@ -48,6 +60,25 @@ namespace DistTestCore
             }
             return container.ClusterExternalAddress;
         }
+
+        private static string GetEnvVarOrDefault(string varName, string defaultValue)
+        {
+            var v = Environment.GetEnvironmentVariable(varName);
+            if (v == null) return defaultValue;
+            return v;
+        }
+
+        private static string? GetNullableEnvVarOrDefault(string varName, string? defaultValue)
+        {
+            var v = Environment.GetEnvironmentVariable(varName);
+            if (v == null) return defaultValue;
+            return v;
+        }
+
+        private static T ParseEnum<T>(string value)
+        {
+            return (T)Enum.Parse(typeof(T), value, true);
+        }
     }
 
     public enum TestRunnerLocation

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -138,7 +138,7 @@ namespace DistTestCore.Helpers
                 if (peer == null) return $"peerId: {node.peerId} is not known.";
 
                 var n = (OnlineCodexNode)peer.Node;
-                var ip = n.CodexAccess.Container.Pod.Ip;
+                var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
                 var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
                 return $"{ip}:{discPort.Number}";
             }

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace
         private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer)
         {
             var startupConfig = new StartupConfig();
-            var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
+            var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.PodInfo.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
             startupConfig.Add(contractsConfig);
             return startupConfig;
         }

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -58,7 +58,7 @@ namespace DistTestCore.Marketplace
             var httpPort = AddExposedPort(tag: HttpPortTag);
 
             var bootPubKey = config.BootstrapNode.PubKey;
-            var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip;
+            var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.PodInfo.Ip;
             var bootPort = config.BootstrapNode.DiscoveryPort.Number;
             var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}";
 

DistTestCore/Metrics/MetricsQuery.cs

@@ -119,7 +119,7 @@ namespace DistTestCore.Metrics
 
         private string GetInstanceNameForNode(RunningContainer node)
         {
-            var ip = node.Pod.Ip;
+            var ip = node.Pod.PodInfo.Ip;
             var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
             return $"{ip}:{port}";
         }

DistTestCore/OnlineCodexNode.cs

@@ -124,7 +124,7 @@ namespace DistTestCore
 
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.Ip);
+            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip);
         }
 
         private void DownloadToFile(string contentId, TestFile file)

DistTestCore/PrometheusStarter.cs

@@ -44,7 +44,7 @@ namespace DistTestCore
 
             foreach (var node in nodes)
             {
-                var ip = node.Pod.Ip;
+                var ip = node.Pod.PodInfo.Ip;
                 var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
                 config += $"          - '{ip}:{port}'\n";
             }

KubernetesWorkflow/ApplicationLifecycle.cs

@@ -1,40 +0,0 @@
-using Utils;
-
-namespace KubernetesWorkflow
-{
-    public class ApplicationLifecycle
-    {
-        private static object instanceLock = new object();
-        private static ApplicationLifecycle? instance;
-        private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
-        private readonly NumberSource namespaceNumberSource = new NumberSource(0);
-
-        private ApplicationLifecycle()
-        {
-        }
-
-        public static ApplicationLifecycle Instance
-        {
-            // I know singletons are quite evil. But we need to be sure this object is created only once
-            // and persists for the entire application lifecycle.
-            get
-            {
-                lock (instanceLock)
-                {
-                    if (instance == null) instance = new ApplicationLifecycle();
-                    return instance;
-                }
-            }
-        }
-
-        public NumberSource GetServiceNumberSource()
-        {
-            return servicePortNumberSource;
-        }
-
-        public string GetTestNamespace()
-        {
-            return namespaceNumberSource.GetNextNumber().ToString("D5");
-        }
-    }
-}

KubernetesWorkflow/CommandRunner.cs

@@ -28,7 +28,7 @@ namespace KubernetesWorkflow
             var input = new[] { command }.Concat(arguments).ToArray();
 
             Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
-                pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
+                pod.PodInfo.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
         }
 
         public string GetStdOut()

KubernetesWorkflow/Configuration.cs

@@ -2,31 +2,17 @@
 {
     public class Configuration
     {
-        public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
+        public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay)
         {
             K8sNamespacePrefix = k8sNamespacePrefix;
             KubeConfigFile = kubeConfigFile;
             OperationTimeout = operationTimeout;
             RetryDelay = retryDelay;
-            LocationMap = locationMap;
         }
 
         public string K8sNamespacePrefix { get; }
         public string? KubeConfigFile { get; }
         public TimeSpan OperationTimeout { get; }
         public TimeSpan RetryDelay { get; }
-        public ConfigurationLocationEntry[] LocationMap { get; }
-    }
-
-    public class ConfigurationLocationEntry
-    {
-        public ConfigurationLocationEntry(Location location, string workerName)
-        {
-            Location = location;
-            WorkerName = workerName;
-        }
-
-        public Location Location { get; }
-        public string WorkerName { get; }
     }
 }

KubernetesWorkflow/K8sCluster.cs

@@ -11,6 +11,7 @@ namespace KubernetesWorkflow
        
         public Configuration Configuration { get; }
         public string HostAddress { get; private set; } = string.Empty;
+        public K8sNodeLabel[] AvailableK8sNodes { get; set; } = new K8sNodeLabel[0];
 
         public KubernetesClientConfiguration GetK8sClientConfig()
         {
@@ -19,10 +20,18 @@ namespace KubernetesWorkflow
             return config;
         }
 
-        public string GetNodeLabelForLocation(Location location)
+        public K8sNodeLabel? GetNodeLabelForLocation(Location location)
         {
-            if (location == Location.Unspecified) return string.Empty;
+            switch (location)
-            return Configuration.LocationMap.Single(l => l.Location == location).WorkerName;
+            {
+                case Location.One:
+                    return K8sNodeIfAvailable(0);
+                case Location.Two:
+                    return K8sNodeIfAvailable(1);
+                case Location.Three:
+                    return K8sNodeIfAvailable(2);
+            }
+            return null;
         }
 
         public TimeSpan K8sOperationTimeout()
@@ -59,5 +68,23 @@ namespace KubernetesWorkflow
                 HostAddress = config.Host;
             }
         }
+
+        private K8sNodeLabel? K8sNodeIfAvailable(int index)
+        {
+            if (AvailableK8sNodes.Length <= index) return null;
+            return AvailableK8sNodes[index];
+        }
+    }
+
+    public class K8sNodeLabel
+    {
+        public K8sNodeLabel(string key, string value)
+        {
+            Key = key;
+            Value = value;
+        }
+
+        public string Key { get; }
+        public string Value { get; }
     }
 }

KubernetesWorkflow/K8sController.cs

@@ -32,13 +32,14 @@ namespace KubernetesWorkflow
         public RunningPod BringOnline(ContainerRecipe[] containerRecipes, Location location)
         {
             log.Debug();
+            DiscoverK8sNodes();
             EnsureTestNamespace();
 
             var deploymentName = CreateDeployment(containerRecipes, location);
             var (serviceName, servicePortsMap) = CreateService(containerRecipes);
-            var (podName, podIp) = FetchNewPod();
+            var podInfo = FetchNewPod();
 
-            return new RunningPod(cluster, podName, podIp, deploymentName, serviceName, servicePortsMap);
+            return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap);
         }
 
         public void Stop(RunningPod pod)
@@ -47,13 +48,13 @@ namespace KubernetesWorkflow
             if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName);
             DeleteDeployment(pod.DeploymentName);
             WaitUntilDeploymentOffline(pod.DeploymentName);
-            WaitUntilPodOffline(pod.Name);
+            WaitUntilPodOffline(pod.PodInfo.Name);
         }
 
         public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
         {
             log.Debug();
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name));
             logHandler.Log(stream);
         }
 
@@ -106,6 +107,42 @@ namespace KubernetesWorkflow
             }
         }
 
+        #region Discover K8s Nodes
+
+        private void DiscoverK8sNodes()
+        {
+            if (cluster.AvailableK8sNodes == null || !cluster.AvailableK8sNodes.Any())
+            {
+                cluster.AvailableK8sNodes = GetAvailableK8sNodes();
+                if (cluster.AvailableK8sNodes.Length < 3)
+                {
+                    log.Debug($"Warning: For full location support, at least 3 Kubernetes Nodes are required in the cluster. Nodes found: '{string.Join(",", cluster.AvailableK8sNodes.Select(p => $"{p.Key}={p.Value}"))}'.");
+                }
+            }
+        }
+
+        private K8sNodeLabel[] GetAvailableK8sNodes()
+        {
+            var nodes = client.Run(c => c.ListNode());
+
+            var optionals = nodes.Items.Select(i => CreateNodeLabel(i));
+            return optionals.Where(n => n != null).Select(n => n!).ToArray();
+        }
+
+        private K8sNodeLabel? CreateNodeLabel(V1Node i)
+        {
+            var keys = i.Metadata.Labels.Keys;
+            var hostnameKey = keys.SingleOrDefault(k => k.ToLowerInvariant().Contains("hostname"));
+            if (hostnameKey != null)
+            {
+                var hostnameValue = i.Metadata.Labels[hostnameKey];
+                return new K8sNodeLabel(hostnameKey, hostnameValue);
+            }
+            return null;
+        }
+
+        #endregion
+
         #region Namespace management
 
         private string K8sTestNamespace { get; }
@@ -314,11 +351,12 @@ namespace KubernetesWorkflow
 
         private IDictionary<string, string> CreateNodeSelector(Location location)
         {
-            if (location == Location.Unspecified) return new Dictionary<string, string>();
+            var nodeLabel = cluster.GetNodeLabelForLocation(location);
+            if (nodeLabel == null) return new Dictionary<string, string>();
 
             return new Dictionary<string, string>
             {
-                { "codex-test-location", cluster.GetNodeLabelForLocation(location) }
+                { nodeLabel.Key, nodeLabel.Value }
             };
         }
 
@@ -401,7 +439,7 @@ namespace KubernetesWorkflow
         {
             var result = new Dictionary<ContainerRecipe, Port[]>();
 
-            var ports = CreateServicePorts(result, containerRecipes);
+            var ports = CreateServicePorts(containerRecipes);
 
             if (!ports.Any())
             {
@@ -424,9 +462,40 @@ namespace KubernetesWorkflow
 
             client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace));
 
+            ReadBackServiceAndMapPorts(serviceSpec, containerRecipes, result);
+
             return (serviceSpec.Metadata.Name, result);
         }
 
+        private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, Dictionary<ContainerRecipe, Port[]> result)
+        {
+            // For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
+            var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace));
+            foreach (var r in containerRecipes)
+            {
+                if (r.ExposedPorts.Any())
+                {
+                    var firstExposedPort = r.ExposedPorts.First();
+                    var portName = GetNameForPort(r, firstExposedPort);
+
+                    var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName);
+                    if (matchingServicePorts.Any())
+                    {
+                        // These service ports belongs to this recipe.
+                        var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
+                        var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
+                        result.Add(r, ports);
+                    }
+                }
+            }
+        }
+
+        private Port? MapNodePortIfAble(V1ServicePort p, string tag)
+        {
+            if (p.NodePort == null) return null;
+            return new Port(p.NodePort.Value, tag);
+        }
+
         private void DeleteService(string serviceName)
         {
             client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace));
@@ -441,36 +510,30 @@ namespace KubernetesWorkflow
             };
         }
 
-        private List<V1ServicePort> CreateServicePorts(Dictionary<ContainerRecipe, Port[]> servicePorts, ContainerRecipe[] recipes)
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes)
         {
             var result = new List<V1ServicePort>();
             foreach (var recipe in recipes)
             {
-                result.AddRange(CreateServicePorts(servicePorts, recipe));
+                result.AddRange(CreateServicePorts(recipe));
             }
             return result;
         }
 
-        private List<V1ServicePort> CreateServicePorts(Dictionary<ContainerRecipe, Port[]> servicePorts, ContainerRecipe recipe)
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe)
         {
             var result = new List<V1ServicePort>();
-            var usedPorts = new List<Port>();
             foreach (var port in recipe.ExposedPorts)
             {
-                var servicePort = workflowNumberSource.GetServicePort();
-                usedPorts.Add(new Port(servicePort, ""));
-
                 result.Add(new V1ServicePort
                 {
                     Name = GetNameForPort(recipe, port),
                     Protocol = "TCP",
                     Port = port.Number,
                     TargetPort = GetNameForPort(recipe, port),
-                    NodePort = servicePort
                 });                
             }
 
-            servicePorts.Add(recipe, usedPorts.ToArray());
             return result;
         }
 
@@ -537,7 +600,7 @@ namespace KubernetesWorkflow
 
         #endregion
 
-        private (string, string) FetchNewPod()
+        private PodInfo FetchNewPod()
         {
             var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
 
@@ -547,12 +610,13 @@ namespace KubernetesWorkflow
             var newPod = newPods.Single();
             var name = newPod.Name();
             var ip = newPod.Status.PodIP;
+            var k8sNodeName = newPod.Spec.NodeName;
 
             if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure.");
             if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure.");
 
             knownPods.Add(name);
-            return (name, ip);
+            return new PodInfo(name, ip, k8sNodeName);
         }
     }
 }

KubernetesWorkflow/Location.cs

@@ -3,7 +3,8 @@
     public enum Location
     {
         Unspecified,
-        BensLaptop,
+        One,
-        BensOldGamingMachine
+        Two,
+        Three,
     }
 }

KubernetesWorkflow/RunningPod.cs

@@ -4,25 +4,38 @@
     {
         private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap;
 
-        public RunningPod(K8sCluster cluster, string name, string ip, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
+        public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
         {
             Cluster = cluster;
-            Name = name;
+            PodInfo = podInfo;
-            Ip = ip;
             DeploymentName = deploymentName;
             ServiceName = serviceName;
             this.servicePortMap = servicePortMap;
         }
 
         public K8sCluster Cluster { get; }
-        public string Name { get; }
+        public PodInfo PodInfo { get; }
-        public string Ip { get; }
         internal string DeploymentName { get; }
         internal string ServiceName { get; }
 
         public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
         {
+            if (!servicePortMap.ContainsKey(containerRecipe)) return Array.Empty<Port>();
             return servicePortMap[containerRecipe];
         }
     }
+
+    public class PodInfo
+    {
+        public PodInfo(string podName, string podIp, string k8sNodeName)
+        {
+            Name = podName;
+            Ip = podIp;
+            K8SNodeName = k8sNodeName;
+        }
+
+        public string Name { get; }
+        public string Ip { get; }
+        public string K8SNodeName { get; }
+    }
 }

KubernetesWorkflow/WorkflowCreator.cs

@@ -16,13 +16,12 @@ namespace KubernetesWorkflow
         {
             cluster = new K8sCluster(configuration);
             this.log = log;
-            testNamespace = ApplicationLifecycle.Instance.GetTestNamespace();
+            testNamespace = Guid.NewGuid().ToString().ToLowerInvariant();
         }
 
         public StartupWorkflow CreateWorkflow()
         {
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
-                                                                    ApplicationLifecycle.Instance.GetServiceNumberSource(),
                                                                     containerNumberSource);
 
             return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);

KubernetesWorkflow/WorkflowNumberSource.cs

@@ -4,13 +4,11 @@ namespace KubernetesWorkflow
 {
     public class WorkflowNumberSource
     {
-        private readonly NumberSource servicePortNumberSource;
         private readonly NumberSource containerNumberSource;
 
-        public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource, NumberSource containerNumberSource)
+        public WorkflowNumberSource(int workflowNumber, NumberSource containerNumberSource)
         {
             WorkflowNumber = workflowNumber;
-            this.servicePortNumberSource = servicePortNumberSource;
             this.containerNumberSource = containerNumberSource;
         }
 
@@ -20,10 +18,5 @@ namespace KubernetesWorkflow
         {
             return containerNumberSource.GetNextNumber();
         }
-
-        public int GetServicePort()
-        {
-            return servicePortNumberSource.GetNextNumber();
-        }
     }
 }

README.md

@@ -17,6 +17,17 @@ Tests are devided into two assemblies: `/Tests` and `/LongTests`.
 
 TODO: All tests will eventually be running as part of a dedicated CI pipeline and kubernetes cluster. Currently, we're developing these tests and the infra-code to support it by running the whole thing locally.
 
+## Configuration
+Test executing can be configured using the following environment variables.
+| Variable       | Description                                                                                                                                          | Default             |
+|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|
+| KUBECONFIG     | Optional path (abs or rel) to kubeconfig YAML file. When null, uses system default (docker-desktop) kubeconfig if available.                         | (null)              |
+| LOGPATH        | Path (abs or rel) where log files will be saved.                                                                                                     | "CodexTestLogs"     |
+| LOGDEBUG       | When "true", enables additional test-runner debug log output.                                                                                                    | "false"             |
+| DATAFILEPATH   | Path (abs or rel) where temporary test data files will be saved.                                                                                     | "TestDataFiles"     |
+| LOGLEVEL       | Codex log-level. (case-insensitive)                                                                                                                  | "Trace"             |
+| RUNNERLOCATION | Use "ExternalToCluster" when test app is running outside of the k8s cluster. Use "InternalToCluster" when tests are run from inside a pod/container. | "ExternalToCluster" |
+
 ## Test logs
 Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs.
 

Tests/BasicTests/TwoClientTests.cs

@@ -28,11 +28,10 @@ namespace Tests.BasicTests
         }
 
         [Test]
-        [Ignore("Requires Location map to be configured for k8s cluster.")]
         public void TwoClientsTwoLocationsTest()
         {
-            var primary = SetupCodexNode(s => s.At(Location.BensLaptop));
+            var primary = SetupCodexNode(s => s.At(Location.One));
-            var secondary = SetupCodexNode(s => s.At(Location.BensOldGamingMachine));
+            var secondary = SetupCodexNode(s => s.At(Location.Two));
 
             PerformTwoClientTest(primary, secondary);
         }