From e7d059ceed480020b52d4e9cecfabf467b0e271a Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 2 Jun 2023 09:03:46 +0200 Subject: [PATCH 1/7] Makes default configuration overridable from environment variables. --- DistTestCore/Configuration.cs | 46 +++++++++++++++++++++++++++++++---- README.md | 11 +++++++++ 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index b9f21de..f66c39d 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -5,11 +5,28 @@ namespace DistTestCore { public class Configuration { + private readonly string? kubeConfigFile; + private readonly string logPath; + private readonly bool logDebug; + private readonly string dataFilesPath; + private readonly CodexLogLevel codexLogLevel; + private readonly TestRunnerLocation runnerLocation; + + public Configuration() + { + kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null); + logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs"); + logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true"; + dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); + codexLogLevel = ParseEnum(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace))); + runnerLocation = ParseEnum(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster))); + } + public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet) { return new KubernetesWorkflow.Configuration( k8sNamespacePrefix: "ct-", - kubeConfigFile: null, + kubeConfigFile: kubeConfigFile, operationTimeout: timeSet.K8sOperationTimeout(), retryDelay: timeSet.WaitForK8sServiceDelay(), locationMap: new[] @@ -22,22 +39,22 @@ namespace DistTestCore public Logging.LogConfig GetLogConfig() { - return new Logging.LogConfig("CodexTestLogs", debugEnabled: false); + return new Logging.LogConfig(logPath, debugEnabled: logDebug); } public string GetFileManagerFolder() { - return "TestDataFiles"; + return dataFilesPath; } public CodexLogLevel GetCodexLogLevel() { - return CodexLogLevel.Trace; + return codexLogLevel; } public TestRunnerLocation GetTestRunnerLocation() { - return TestRunnerLocation.ExternalToCluster; + return runnerLocation; } public RunningContainerAddress GetAddress(RunningContainer container) @@ -48,6 +65,25 @@ namespace DistTestCore } return container.ClusterExternalAddress; } + + private static string GetEnvVarOrDefault(string varName, string defaultValue) + { + var v = Environment.GetEnvironmentVariable(varName); + if (v == null) return defaultValue; + return v; + } + + private static string? GetNullableEnvVarOrDefault(string varName, string? defaultValue) + { + var v = Environment.GetEnvironmentVariable(varName); + if (v == null) return defaultValue; + return v; + } + + private static T ParseEnum(string value) + { + return (T)Enum.Parse(typeof(T), value, true); + } } public enum TestRunnerLocation diff --git a/README.md b/README.md index 9f5ebef..4d4760c 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,17 @@ Tests are devided into two assemblies: `/Tests` and `/LongTests`. TODO: All tests will eventually be running as part of a dedicated CI pipeline and kubernetes cluster. Currently, we're developing these tests and the infra-code to support it by running the whole thing locally. +## Configuration +Test executing can be configured using the following environment variables. +| Variable | Description | Default | +|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------| +| KUBECONFIG | Optional path (abs or rel) to kubeconfig YAML file. When null, uses system default (docker-desktop) kubeconfig if available. | (null) | +| LOGPATH | Path (abs or rel) where log files will be saved. | "CodexTestLogs" | +| LOGDEBUG | When "true", enables additional test-runner debug log output. | "false" | +| DATAFILEPATH | Path (abs or rel) where temporary test data files will be saved. | "TestDataFiles" | +| LOGLEVEL | Codex log-level. (case-insensitive) | "Trace" | +| RUNNERLOCATION | Use "ExternalToCluster" when test app is running outside of the k8s cluster. Use "InternalToCluster" when tests are run from inside a pod/container. | "ExternalToCluster" | + ## Test logs Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs. From ad71cff4657ca000a37775f87e16ab44cdc37c97 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 2 Jun 2023 10:04:07 +0200 Subject: [PATCH 2/7] Automatically map location enum to available k8s nodes. --- DistTestCore/Codex/CodexContainerRecipe.cs | 2 +- DistTestCore/CodexStarter.cs | 3 +- DistTestCore/Configuration.cs | 7 +--- .../Helpers/PeerConnectionTestHelpers.cs | 2 +- .../Marketplace/CodexContractsStarter.cs | 2 +- .../Marketplace/GethContainerRecipe.cs | 2 +- DistTestCore/Metrics/MetricsQuery.cs | 2 +- DistTestCore/OnlineCodexNode.cs | 2 +- DistTestCore/PrometheusStarter.cs | 2 +- KubernetesWorkflow/CommandRunner.cs | 2 +- KubernetesWorkflow/Configuration.cs | 16 +-------- KubernetesWorkflow/K8sCluster.cs | 19 ++++++++-- KubernetesWorkflow/K8sController.cs | 36 +++++++++++++++---- KubernetesWorkflow/Location.cs | 5 +-- KubernetesWorkflow/RunningPod.cs | 22 +++++++++--- Tests/BasicTests/TwoClientTests.cs | 5 ++- 16 files changed, 81 insertions(+), 48 deletions(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index a723baa..e92c93c 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -49,7 +49,7 @@ namespace DistTestCore.Codex var companionNodeAccount = companionNode.Accounts[Index]; Additional(companionNodeAccount); - var ip = companionNode.RunningContainer.Pod.Ip; + var ip = companionNode.RunningContainer.Pod.PodInfo.Ip; var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number; AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}"); diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 1c49cad..627bfb9 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -28,7 +28,8 @@ namespace DistTestCore var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory); var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); - LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: {group.Describe()}"); + var podInfo = group.Containers.RunningPod.PodInfo; + LogEnd($"Started {codexSetup.NumberOfNodes} nodes at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}"); LogSeparator(); return group; } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index f66c39d..8f39381 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -28,12 +28,7 @@ namespace DistTestCore k8sNamespacePrefix: "ct-", kubeConfigFile: kubeConfigFile, operationTimeout: timeSet.K8sOperationTimeout(), - retryDelay: timeSet.WaitForK8sServiceDelay(), - locationMap: new[] - { - new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"), - new ConfigurationLocationEntry(Location.BensLaptop, "worker02"), - } + retryDelay: timeSet.WaitForK8sServiceDelay() ); } diff --git a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs index 92114bc..54f3c43 100644 --- a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs +++ b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs @@ -138,7 +138,7 @@ namespace DistTestCore.Helpers if (peer == null) return $"peerId: {node.peerId} is not known."; var n = (OnlineCodexNode)peer.Node; - var ip = n.CodexAccess.Container.Pod.Ip; + var ip = n.CodexAccess.Container.Pod.PodInfo.Ip; var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag); return $"{ip}:{discPort.Number}"; } diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index 68d64fe..7a45016 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer) { var startupConfig = new StartupConfig(); - var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag)); + var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.PodInfo.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag)); startupConfig.Add(contractsConfig); return startupConfig; } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index fa95054..a377d69 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -58,7 +58,7 @@ namespace DistTestCore.Marketplace var httpPort = AddExposedPort(tag: HttpPortTag); var bootPubKey = config.BootstrapNode.PubKey; - var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip; + var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.PodInfo.Ip; var bootPort = config.BootstrapNode.DiscoveryPort.Number; var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}"; diff --git a/DistTestCore/Metrics/MetricsQuery.cs b/DistTestCore/Metrics/MetricsQuery.cs index fc19867..baffe66 100644 --- a/DistTestCore/Metrics/MetricsQuery.cs +++ b/DistTestCore/Metrics/MetricsQuery.cs @@ -119,7 +119,7 @@ namespace DistTestCore.Metrics private string GetInstanceNameForNode(RunningContainer node) { - var ip = node.Pod.Ip; + var ip = node.Pod.PodInfo.Ip; var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; return $"{ip}:{port}"; } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 322fb04..6e12c79 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -124,7 +124,7 @@ namespace DistTestCore // The peer we want to connect is in a different pod. // We must replace the default IP with the pod IP in the multiAddress. - return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.Ip); + return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip); } private void DownloadToFile(string contentId, TestFile file) diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index eb66efc..64edae9 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -44,7 +44,7 @@ namespace DistTestCore foreach (var node in nodes) { - var ip = node.Pod.Ip; + var ip = node.Pod.PodInfo.Ip; var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; config += $" - '{ip}:{port}'\n"; } diff --git a/KubernetesWorkflow/CommandRunner.cs b/KubernetesWorkflow/CommandRunner.cs index daf8b9b..7c6b948 100644 --- a/KubernetesWorkflow/CommandRunner.cs +++ b/KubernetesWorkflow/CommandRunner.cs @@ -28,7 +28,7 @@ namespace KubernetesWorkflow var input = new[] { command }.Concat(arguments).ToArray(); Time.Wait(client.Run(c => c.NamespacedPodExecAsync( - pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()))); + pod.PodInfo.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()))); } public string GetStdOut() diff --git a/KubernetesWorkflow/Configuration.cs b/KubernetesWorkflow/Configuration.cs index f94924d..53fee79 100644 --- a/KubernetesWorkflow/Configuration.cs +++ b/KubernetesWorkflow/Configuration.cs @@ -2,31 +2,17 @@ { public class Configuration { - public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap) + public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay) { K8sNamespacePrefix = k8sNamespacePrefix; KubeConfigFile = kubeConfigFile; OperationTimeout = operationTimeout; RetryDelay = retryDelay; - LocationMap = locationMap; } public string K8sNamespacePrefix { get; } public string? KubeConfigFile { get; } public TimeSpan OperationTimeout { get; } public TimeSpan RetryDelay { get; } - public ConfigurationLocationEntry[] LocationMap { get; } - } - - public class ConfigurationLocationEntry - { - public ConfigurationLocationEntry(Location location, string workerName) - { - Location = location; - WorkerName = workerName; - } - - public Location Location { get; } - public string WorkerName { get; } } } diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs index 4048164..ef923a6 100644 --- a/KubernetesWorkflow/K8sCluster.cs +++ b/KubernetesWorkflow/K8sCluster.cs @@ -11,6 +11,7 @@ namespace KubernetesWorkflow public Configuration Configuration { get; } public string HostAddress { get; private set; } = string.Empty; + public string[] AvailableK8sNodes { get; set; } = new string[0]; public KubernetesClientConfiguration GetK8sClientConfig() { @@ -21,8 +22,16 @@ namespace KubernetesWorkflow public string GetNodeLabelForLocation(Location location) { - if (location == Location.Unspecified) return string.Empty; - return Configuration.LocationMap.Single(l => l.Location == location).WorkerName; + switch (location) + { + case Location.One: + return K8sNodeIfAvailable(0); + case Location.Two: + return K8sNodeIfAvailable(1); + case Location.Three: + return K8sNodeIfAvailable(2); + } + return string.Empty; } public TimeSpan K8sOperationTimeout() @@ -59,5 +68,11 @@ namespace KubernetesWorkflow HostAddress = config.Host; } } + + private string K8sNodeIfAvailable(int index) + { + if (AvailableK8sNodes.Length <= index) return string.Empty; + return AvailableK8sNodes[index]; + } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index f094134..16d645a 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -32,13 +32,14 @@ namespace KubernetesWorkflow public RunningPod BringOnline(ContainerRecipe[] containerRecipes, Location location) { log.Debug(); + DiscoverK8sNodes(); EnsureTestNamespace(); var deploymentName = CreateDeployment(containerRecipes, location); var (serviceName, servicePortsMap) = CreateService(containerRecipes); - var (podName, podIp) = FetchNewPod(); + var podInfo = FetchNewPod(); - return new RunningPod(cluster, podName, podIp, deploymentName, serviceName, servicePortsMap); + return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap); } public void Stop(RunningPod pod) @@ -47,13 +48,13 @@ namespace KubernetesWorkflow if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName); DeleteDeployment(pod.DeploymentName); WaitUntilDeploymentOffline(pod.DeploymentName); - WaitUntilPodOffline(pod.Name); + WaitUntilPodOffline(pod.PodInfo.Name); } public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) { log.Debug(); - using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name)); + using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name)); logHandler.Log(stream); } @@ -106,6 +107,28 @@ namespace KubernetesWorkflow } } + #region Discover K8s Nodes + + private void DiscoverK8sNodes() + { + if (cluster.AvailableK8sNodes == null || !cluster.AvailableK8sNodes.Any()) + { + cluster.AvailableK8sNodes = GetAvailableK8sNodes(); + if (cluster.AvailableK8sNodes.Length < 3) + { + log.Debug($"Warning: For full location support, at least 3 Kubernetes Nodes are required in the cluster. Nodes found: '{string.Join(",", cluster.AvailableK8sNodes)}'."); + } + } + } + + private string[] GetAvailableK8sNodes() + { + var nodes = client.Run(c => c.ListNode()); + return nodes.Items.Select(i => i.Metadata.Name).ToArray(); + } + + #endregion + #region Namespace management private string K8sTestNamespace { get; } @@ -537,7 +560,7 @@ namespace KubernetesWorkflow #endregion - private (string, string) FetchNewPod() + private PodInfo FetchNewPod() { var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items; @@ -547,12 +570,13 @@ namespace KubernetesWorkflow var newPod = newPods.Single(); var name = newPod.Name(); var ip = newPod.Status.PodIP; + var k8sNodeName = newPod.Spec.NodeName; if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure."); if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure."); knownPods.Add(name); - return (name, ip); + return new PodInfo(name, ip, k8sNodeName); } } } diff --git a/KubernetesWorkflow/Location.cs b/KubernetesWorkflow/Location.cs index 3b01284..adc2862 100644 --- a/KubernetesWorkflow/Location.cs +++ b/KubernetesWorkflow/Location.cs @@ -3,7 +3,8 @@ public enum Location { Unspecified, - BensLaptop, - BensOldGamingMachine + One, + Two, + Three, } } diff --git a/KubernetesWorkflow/RunningPod.cs b/KubernetesWorkflow/RunningPod.cs index b676903..946d15e 100644 --- a/KubernetesWorkflow/RunningPod.cs +++ b/KubernetesWorkflow/RunningPod.cs @@ -4,19 +4,17 @@ { private readonly Dictionary servicePortMap; - public RunningPod(K8sCluster cluster, string name, string ip, string deploymentName, string serviceName, Dictionary servicePortMap) + public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary servicePortMap) { Cluster = cluster; - Name = name; - Ip = ip; + PodInfo = podInfo; DeploymentName = deploymentName; ServiceName = serviceName; this.servicePortMap = servicePortMap; } public K8sCluster Cluster { get; } - public string Name { get; } - public string Ip { get; } + public PodInfo PodInfo { get; } internal string DeploymentName { get; } internal string ServiceName { get; } @@ -25,4 +23,18 @@ return servicePortMap[containerRecipe]; } } + + public class PodInfo + { + public PodInfo(string podName, string podIp, string k8sNodeName) + { + Name = podName; + Ip = podIp; + K8SNodeName = k8sNodeName; + } + + public string Name { get; } + public string Ip { get; } + public string K8SNodeName { get; } + } } diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs index d4edbed..d03633b 100644 --- a/Tests/BasicTests/TwoClientTests.cs +++ b/Tests/BasicTests/TwoClientTests.cs @@ -28,11 +28,10 @@ namespace Tests.BasicTests } [Test] - [Ignore("Requires Location map to be configured for k8s cluster.")] public void TwoClientsTwoLocationsTest() { - var primary = SetupCodexNode(s => s.At(Location.BensLaptop)); - var secondary = SetupCodexNode(s => s.At(Location.BensOldGamingMachine)); + var primary = SetupCodexNode(s => s.At(Location.One)); + var secondary = SetupCodexNode(s => s.At(Location.Two)); PerformTwoClientTest(primary, secondary); } From cda63ba245bde251ffe57a08e883e139c0460d97 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 2 Jun 2023 10:27:57 +0200 Subject: [PATCH 3/7] Dynamically allocates Locations-enum to available k8s nodes in cluster. --- KubernetesWorkflow/K8sCluster.cs | 22 +++++++++++++++++----- KubernetesWorkflow/K8sController.cs | 25 ++++++++++++++++++++----- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs index ef923a6..665b87f 100644 --- a/KubernetesWorkflow/K8sCluster.cs +++ b/KubernetesWorkflow/K8sCluster.cs @@ -11,7 +11,7 @@ namespace KubernetesWorkflow public Configuration Configuration { get; } public string HostAddress { get; private set; } = string.Empty; - public string[] AvailableK8sNodes { get; set; } = new string[0]; + public K8sNodeLabel[] AvailableK8sNodes { get; set; } = new K8sNodeLabel[0]; public KubernetesClientConfiguration GetK8sClientConfig() { @@ -20,7 +20,7 @@ namespace KubernetesWorkflow return config; } - public string GetNodeLabelForLocation(Location location) + public K8sNodeLabel? GetNodeLabelForLocation(Location location) { switch (location) { @@ -31,7 +31,7 @@ namespace KubernetesWorkflow case Location.Three: return K8sNodeIfAvailable(2); } - return string.Empty; + return null; } public TimeSpan K8sOperationTimeout() @@ -69,10 +69,22 @@ namespace KubernetesWorkflow } } - private string K8sNodeIfAvailable(int index) + private K8sNodeLabel? K8sNodeIfAvailable(int index) { - if (AvailableK8sNodes.Length <= index) return string.Empty; + if (AvailableK8sNodes.Length <= index) return null; return AvailableK8sNodes[index]; } } + + public class K8sNodeLabel + { + public K8sNodeLabel(string key, string value) + { + Key = key; + Value = value; + } + + public string Key { get; } + public string Value { get; } + } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 16d645a..803fea0 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -116,15 +116,29 @@ namespace KubernetesWorkflow cluster.AvailableK8sNodes = GetAvailableK8sNodes(); if (cluster.AvailableK8sNodes.Length < 3) { - log.Debug($"Warning: For full location support, at least 3 Kubernetes Nodes are required in the cluster. Nodes found: '{string.Join(",", cluster.AvailableK8sNodes)}'."); + log.Debug($"Warning: For full location support, at least 3 Kubernetes Nodes are required in the cluster. Nodes found: '{string.Join(",", cluster.AvailableK8sNodes.Select(p => $"{p.Key}={p.Value}"))}'."); } } } - private string[] GetAvailableK8sNodes() + private K8sNodeLabel[] GetAvailableK8sNodes() { var nodes = client.Run(c => c.ListNode()); - return nodes.Items.Select(i => i.Metadata.Name).ToArray(); + + var optionals = nodes.Items.Select(i => CreateNodeLabel(i)); + return optionals.Where(n => n != null).Select(n => n!).ToArray(); + } + + private K8sNodeLabel? CreateNodeLabel(V1Node i) + { + var keys = i.Metadata.Labels.Keys; + var hostnameKey = keys.SingleOrDefault(k => k.ToLowerInvariant().Contains("hostname")); + if (hostnameKey != null) + { + var hostnameValue = i.Metadata.Labels[hostnameKey]; + return new K8sNodeLabel(hostnameKey, hostnameValue); + } + return null; } #endregion @@ -337,11 +351,12 @@ namespace KubernetesWorkflow private IDictionary CreateNodeSelector(Location location) { - if (location == Location.Unspecified) return new Dictionary(); + var nodeLabel = cluster.GetNodeLabelForLocation(location); + if (nodeLabel == null) return new Dictionary(); return new Dictionary { - { "codex-test-location", cluster.GetNodeLabelForLocation(location) } + { nodeLabel.Key, nodeLabel.Value } }; } From 8ba4b1a2907fc4661ffc56b8e1613667f3041081 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 2 Jun 2023 11:07:36 +0200 Subject: [PATCH 4/7] Removes pre-assigned service ports and reads back service-ports assigned by k8s cluster. --- KubernetesWorkflow/ApplicationLifecycle.cs | 6 --- KubernetesWorkflow/K8sController.cs | 45 +++++++++++++++++----- KubernetesWorkflow/WorkflowCreator.cs | 1 - KubernetesWorkflow/WorkflowNumberSource.cs | 9 +---- 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/KubernetesWorkflow/ApplicationLifecycle.cs b/KubernetesWorkflow/ApplicationLifecycle.cs index 7d6fb3d..b14fe52 100644 --- a/KubernetesWorkflow/ApplicationLifecycle.cs +++ b/KubernetesWorkflow/ApplicationLifecycle.cs @@ -6,7 +6,6 @@ namespace KubernetesWorkflow { private static object instanceLock = new object(); private static ApplicationLifecycle? instance; - private readonly NumberSource servicePortNumberSource = new NumberSource(30001); private readonly NumberSource namespaceNumberSource = new NumberSource(0); private ApplicationLifecycle() @@ -27,11 +26,6 @@ namespace KubernetesWorkflow } } - public NumberSource GetServiceNumberSource() - { - return servicePortNumberSource; - } - public string GetTestNamespace() { return namespaceNumberSource.GetNextNumber().ToString("D5"); diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 803fea0..e0f8627 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -439,7 +439,7 @@ namespace KubernetesWorkflow { var result = new Dictionary(); - var ports = CreateServicePorts(result, containerRecipes); + var ports = CreateServicePorts(containerRecipes); if (!ports.Any()) { @@ -462,9 +462,40 @@ namespace KubernetesWorkflow client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace)); + ReadBackServiceAndMapPorts(serviceSpec, containerRecipes, result); + return (serviceSpec.Metadata.Name, result); } + private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, Dictionary result) + { + // For each container-recipe, we need to figure out which service-ports it was assigned by K8s. + var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace)); + foreach (var r in containerRecipes) + { + if (r.ExposedPorts.Any()) + { + var firstExposedPort = r.ExposedPorts.First(); + var portName = GetNameForPort(r, firstExposedPort); + + var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName); + if (matchingServicePorts.Any()) + { + // These service ports belongs to this recipe. + var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName)); + var ports = optionals.Where(p => p != null).Select(p => p!).ToArray(); + result.Add(r, ports); + } + } + } + } + + private Port? MapNodePortIfAble(V1ServicePort p, string tag) + { + if (p.NodePort == null) return null; + return new Port(p.NodePort.Value, tag); + } + private void DeleteService(string serviceName) { client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace)); @@ -479,36 +510,30 @@ namespace KubernetesWorkflow }; } - private List CreateServicePorts(Dictionary servicePorts, ContainerRecipe[] recipes) + private List CreateServicePorts(ContainerRecipe[] recipes) { var result = new List(); foreach (var recipe in recipes) { - result.AddRange(CreateServicePorts(servicePorts, recipe)); + result.AddRange(CreateServicePorts(recipe)); } return result; } - private List CreateServicePorts(Dictionary servicePorts, ContainerRecipe recipe) + private List CreateServicePorts(ContainerRecipe recipe) { var result = new List(); - var usedPorts = new List(); foreach (var port in recipe.ExposedPorts) { - var servicePort = workflowNumberSource.GetServicePort(); - usedPorts.Add(new Port(servicePort, "")); - result.Add(new V1ServicePort { Name = GetNameForPort(recipe, port), Protocol = "TCP", Port = port.Number, TargetPort = GetNameForPort(recipe, port), - NodePort = servicePort }); } - servicePorts.Add(recipe, usedPorts.ToArray()); return result; } diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index aa0a098..3a707cc 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -22,7 +22,6 @@ namespace KubernetesWorkflow public StartupWorkflow CreateWorkflow() { var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(), - ApplicationLifecycle.Instance.GetServiceNumberSource(), containerNumberSource); return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace); diff --git a/KubernetesWorkflow/WorkflowNumberSource.cs b/KubernetesWorkflow/WorkflowNumberSource.cs index 8cbab34..cf1e53e 100644 --- a/KubernetesWorkflow/WorkflowNumberSource.cs +++ b/KubernetesWorkflow/WorkflowNumberSource.cs @@ -4,13 +4,11 @@ namespace KubernetesWorkflow { public class WorkflowNumberSource { - private readonly NumberSource servicePortNumberSource; private readonly NumberSource containerNumberSource; - public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource, NumberSource containerNumberSource) + public WorkflowNumberSource(int workflowNumber, NumberSource containerNumberSource) { WorkflowNumber = workflowNumber; - this.servicePortNumberSource = servicePortNumberSource; this.containerNumberSource = containerNumberSource; } @@ -20,10 +18,5 @@ namespace KubernetesWorkflow { return containerNumberSource.GetNextNumber(); } - - public int GetServicePort() - { - return servicePortNumberSource.GetNextNumber(); - } } } From 303eb99f490dd1e2c167f8284e01fc12feee2ba3 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 2 Jun 2023 11:10:16 +0200 Subject: [PATCH 5/7] Removes application-level managed namespaces. Use GUIDs instead. --- KubernetesWorkflow/ApplicationLifecycle.cs | 34 ---------------------- KubernetesWorkflow/WorkflowCreator.cs | 2 +- 2 files changed, 1 insertion(+), 35 deletions(-) delete mode 100644 KubernetesWorkflow/ApplicationLifecycle.cs diff --git a/KubernetesWorkflow/ApplicationLifecycle.cs b/KubernetesWorkflow/ApplicationLifecycle.cs deleted file mode 100644 index b14fe52..0000000 --- a/KubernetesWorkflow/ApplicationLifecycle.cs +++ /dev/null @@ -1,34 +0,0 @@ -using Utils; - -namespace KubernetesWorkflow -{ - public class ApplicationLifecycle - { - private static object instanceLock = new object(); - private static ApplicationLifecycle? instance; - private readonly NumberSource namespaceNumberSource = new NumberSource(0); - - private ApplicationLifecycle() - { - } - - public static ApplicationLifecycle Instance - { - // I know singletons are quite evil. But we need to be sure this object is created only once - // and persists for the entire application lifecycle. - get - { - lock (instanceLock) - { - if (instance == null) instance = new ApplicationLifecycle(); - return instance; - } - } - } - - public string GetTestNamespace() - { - return namespaceNumberSource.GetNextNumber().ToString("D5"); - } - } -} diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index 3a707cc..d03dca7 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -16,7 +16,7 @@ namespace KubernetesWorkflow { cluster = new K8sCluster(configuration); this.log = log; - testNamespace = ApplicationLifecycle.Instance.GetTestNamespace(); + testNamespace = Guid.NewGuid().ToString().ToLowerInvariant(); } public StartupWorkflow CreateWorkflow() From 346a63abaa356de4e58900027583e583db4491dd Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 2 Jun 2023 11:30:03 +0200 Subject: [PATCH 6/7] Fixes crash for container-recipes that expose no external ports. --- KubernetesWorkflow/RunningPod.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/KubernetesWorkflow/RunningPod.cs b/KubernetesWorkflow/RunningPod.cs index 946d15e..bee0df9 100644 --- a/KubernetesWorkflow/RunningPod.cs +++ b/KubernetesWorkflow/RunningPod.cs @@ -20,6 +20,7 @@ public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe) { + if (servicePortMap.ContainsKey(containerRecipe)) return Array.Empty(); return servicePortMap[containerRecipe]; } } From 7c4a2ea12ce34b4f48dad6fe5530eb34aa168554 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 2 Jun 2023 11:38:52 +0200 Subject: [PATCH 7/7] Fixes the fix of the failure for container-recipes with no exposed ports. --- KubernetesWorkflow/RunningPod.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/KubernetesWorkflow/RunningPod.cs b/KubernetesWorkflow/RunningPod.cs index bee0df9..1618410 100644 --- a/KubernetesWorkflow/RunningPod.cs +++ b/KubernetesWorkflow/RunningPod.cs @@ -20,7 +20,7 @@ public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe) { - if (servicePortMap.ContainsKey(containerRecipe)) return Array.Empty(); + if (!servicePortMap.ContainsKey(containerRecipe)) return Array.Empty(); return servicePortMap[containerRecipe]; } }