From ad71cff4657ca000a37775f87e16ab44cdc37c97 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 2 Jun 2023 10:04:07 +0200 Subject: [PATCH] Automatically map location enum to available k8s nodes. --- DistTestCore/Codex/CodexContainerRecipe.cs | 2 +- DistTestCore/CodexStarter.cs | 3 +- DistTestCore/Configuration.cs | 7 +--- .../Helpers/PeerConnectionTestHelpers.cs | 2 +- .../Marketplace/CodexContractsStarter.cs | 2 +- .../Marketplace/GethContainerRecipe.cs | 2 +- DistTestCore/Metrics/MetricsQuery.cs | 2 +- DistTestCore/OnlineCodexNode.cs | 2 +- DistTestCore/PrometheusStarter.cs | 2 +- KubernetesWorkflow/CommandRunner.cs | 2 +- KubernetesWorkflow/Configuration.cs | 16 +-------- KubernetesWorkflow/K8sCluster.cs | 19 ++++++++-- KubernetesWorkflow/K8sController.cs | 36 +++++++++++++++---- KubernetesWorkflow/Location.cs | 5 +-- KubernetesWorkflow/RunningPod.cs | 22 +++++++++--- Tests/BasicTests/TwoClientTests.cs | 5 ++- 16 files changed, 81 insertions(+), 48 deletions(-) diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index a723baa..e92c93c 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -49,7 +49,7 @@ namespace DistTestCore.Codex var companionNodeAccount = companionNode.Accounts[Index]; Additional(companionNodeAccount); - var ip = companionNode.RunningContainer.Pod.Ip; + var ip = companionNode.RunningContainer.Pod.PodInfo.Ip; var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number; AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}"); diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 1c49cad..627bfb9 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -28,7 +28,8 @@ namespace DistTestCore var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory); var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); - LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: {group.Describe()}"); + var podInfo = group.Containers.RunningPod.PodInfo; + LogEnd($"Started {codexSetup.NumberOfNodes} nodes at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}"); LogSeparator(); return group; } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index f66c39d..8f39381 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -28,12 +28,7 @@ namespace DistTestCore k8sNamespacePrefix: "ct-", kubeConfigFile: kubeConfigFile, operationTimeout: timeSet.K8sOperationTimeout(), - retryDelay: timeSet.WaitForK8sServiceDelay(), - locationMap: new[] - { - new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"), - new ConfigurationLocationEntry(Location.BensLaptop, "worker02"), - } + retryDelay: timeSet.WaitForK8sServiceDelay() ); } diff --git a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs index 92114bc..54f3c43 100644 --- a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs +++ b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs @@ -138,7 +138,7 @@ namespace DistTestCore.Helpers if (peer == null) return $"peerId: {node.peerId} is not known."; var n = (OnlineCodexNode)peer.Node; - var ip = n.CodexAccess.Container.Pod.Ip; + var ip = n.CodexAccess.Container.Pod.PodInfo.Ip; var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag); return $"{ip}:{discPort.Number}"; } diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index 68d64fe..7a45016 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer) { var startupConfig = new StartupConfig(); - var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag)); + var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.PodInfo.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag)); startupConfig.Add(contractsConfig); return startupConfig; } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index fa95054..a377d69 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -58,7 +58,7 @@ namespace DistTestCore.Marketplace var httpPort = AddExposedPort(tag: HttpPortTag); var bootPubKey = config.BootstrapNode.PubKey; - var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip; + var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.PodInfo.Ip; var bootPort = config.BootstrapNode.DiscoveryPort.Number; var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}"; diff --git a/DistTestCore/Metrics/MetricsQuery.cs b/DistTestCore/Metrics/MetricsQuery.cs index fc19867..baffe66 100644 --- a/DistTestCore/Metrics/MetricsQuery.cs +++ b/DistTestCore/Metrics/MetricsQuery.cs @@ -119,7 +119,7 @@ namespace DistTestCore.Metrics private string GetInstanceNameForNode(RunningContainer node) { - var ip = node.Pod.Ip; + var ip = node.Pod.PodInfo.Ip; var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; return $"{ip}:{port}"; } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 322fb04..6e12c79 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -124,7 +124,7 @@ namespace DistTestCore // The peer we want to connect is in a different pod. // We must replace the default IP with the pod IP in the multiAddress. - return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.Ip); + return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip); } private void DownloadToFile(string contentId, TestFile file) diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index eb66efc..64edae9 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -44,7 +44,7 @@ namespace DistTestCore foreach (var node in nodes) { - var ip = node.Pod.Ip; + var ip = node.Pod.PodInfo.Ip; var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number; config += $" - '{ip}:{port}'\n"; } diff --git a/KubernetesWorkflow/CommandRunner.cs b/KubernetesWorkflow/CommandRunner.cs index daf8b9b..7c6b948 100644 --- a/KubernetesWorkflow/CommandRunner.cs +++ b/KubernetesWorkflow/CommandRunner.cs @@ -28,7 +28,7 @@ namespace KubernetesWorkflow var input = new[] { command }.Concat(arguments).ToArray(); Time.Wait(client.Run(c => c.NamespacedPodExecAsync( - pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()))); + pod.PodInfo.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()))); } public string GetStdOut() diff --git a/KubernetesWorkflow/Configuration.cs b/KubernetesWorkflow/Configuration.cs index f94924d..53fee79 100644 --- a/KubernetesWorkflow/Configuration.cs +++ b/KubernetesWorkflow/Configuration.cs @@ -2,31 +2,17 @@ { public class Configuration { - public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap) + public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay) { K8sNamespacePrefix = k8sNamespacePrefix; KubeConfigFile = kubeConfigFile; OperationTimeout = operationTimeout; RetryDelay = retryDelay; - LocationMap = locationMap; } public string K8sNamespacePrefix { get; } public string? KubeConfigFile { get; } public TimeSpan OperationTimeout { get; } public TimeSpan RetryDelay { get; } - public ConfigurationLocationEntry[] LocationMap { get; } - } - - public class ConfigurationLocationEntry - { - public ConfigurationLocationEntry(Location location, string workerName) - { - Location = location; - WorkerName = workerName; - } - - public Location Location { get; } - public string WorkerName { get; } } } diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs index 4048164..ef923a6 100644 --- a/KubernetesWorkflow/K8sCluster.cs +++ b/KubernetesWorkflow/K8sCluster.cs @@ -11,6 +11,7 @@ namespace KubernetesWorkflow public Configuration Configuration { get; } public string HostAddress { get; private set; } = string.Empty; + public string[] AvailableK8sNodes { get; set; } = new string[0]; public KubernetesClientConfiguration GetK8sClientConfig() { @@ -21,8 +22,16 @@ namespace KubernetesWorkflow public string GetNodeLabelForLocation(Location location) { - if (location == Location.Unspecified) return string.Empty; - return Configuration.LocationMap.Single(l => l.Location == location).WorkerName; + switch (location) + { + case Location.One: + return K8sNodeIfAvailable(0); + case Location.Two: + return K8sNodeIfAvailable(1); + case Location.Three: + return K8sNodeIfAvailable(2); + } + return string.Empty; } public TimeSpan K8sOperationTimeout() @@ -59,5 +68,11 @@ namespace KubernetesWorkflow HostAddress = config.Host; } } + + private string K8sNodeIfAvailable(int index) + { + if (AvailableK8sNodes.Length <= index) return string.Empty; + return AvailableK8sNodes[index]; + } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index f094134..16d645a 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -32,13 +32,14 @@ namespace KubernetesWorkflow public RunningPod BringOnline(ContainerRecipe[] containerRecipes, Location location) { log.Debug(); + DiscoverK8sNodes(); EnsureTestNamespace(); var deploymentName = CreateDeployment(containerRecipes, location); var (serviceName, servicePortsMap) = CreateService(containerRecipes); - var (podName, podIp) = FetchNewPod(); + var podInfo = FetchNewPod(); - return new RunningPod(cluster, podName, podIp, deploymentName, serviceName, servicePortsMap); + return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap); } public void Stop(RunningPod pod) @@ -47,13 +48,13 @@ namespace KubernetesWorkflow if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName); DeleteDeployment(pod.DeploymentName); WaitUntilDeploymentOffline(pod.DeploymentName); - WaitUntilPodOffline(pod.Name); + WaitUntilPodOffline(pod.PodInfo.Name); } public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) { log.Debug(); - using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name)); + using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name)); logHandler.Log(stream); } @@ -106,6 +107,28 @@ namespace KubernetesWorkflow } } + #region Discover K8s Nodes + + private void DiscoverK8sNodes() + { + if (cluster.AvailableK8sNodes == null || !cluster.AvailableK8sNodes.Any()) + { + cluster.AvailableK8sNodes = GetAvailableK8sNodes(); + if (cluster.AvailableK8sNodes.Length < 3) + { + log.Debug($"Warning: For full location support, at least 3 Kubernetes Nodes are required in the cluster. Nodes found: '{string.Join(",", cluster.AvailableK8sNodes)}'."); + } + } + } + + private string[] GetAvailableK8sNodes() + { + var nodes = client.Run(c => c.ListNode()); + return nodes.Items.Select(i => i.Metadata.Name).ToArray(); + } + + #endregion + #region Namespace management private string K8sTestNamespace { get; } @@ -537,7 +560,7 @@ namespace KubernetesWorkflow #endregion - private (string, string) FetchNewPod() + private PodInfo FetchNewPod() { var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items; @@ -547,12 +570,13 @@ namespace KubernetesWorkflow var newPod = newPods.Single(); var name = newPod.Name(); var ip = newPod.Status.PodIP; + var k8sNodeName = newPod.Spec.NodeName; if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure."); if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure."); knownPods.Add(name); - return (name, ip); + return new PodInfo(name, ip, k8sNodeName); } } } diff --git a/KubernetesWorkflow/Location.cs b/KubernetesWorkflow/Location.cs index 3b01284..adc2862 100644 --- a/KubernetesWorkflow/Location.cs +++ b/KubernetesWorkflow/Location.cs @@ -3,7 +3,8 @@ public enum Location { Unspecified, - BensLaptop, - BensOldGamingMachine + One, + Two, + Three, } } diff --git a/KubernetesWorkflow/RunningPod.cs b/KubernetesWorkflow/RunningPod.cs index b676903..946d15e 100644 --- a/KubernetesWorkflow/RunningPod.cs +++ b/KubernetesWorkflow/RunningPod.cs @@ -4,19 +4,17 @@ { private readonly Dictionary servicePortMap; - public RunningPod(K8sCluster cluster, string name, string ip, string deploymentName, string serviceName, Dictionary servicePortMap) + public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary servicePortMap) { Cluster = cluster; - Name = name; - Ip = ip; + PodInfo = podInfo; DeploymentName = deploymentName; ServiceName = serviceName; this.servicePortMap = servicePortMap; } public K8sCluster Cluster { get; } - public string Name { get; } - public string Ip { get; } + public PodInfo PodInfo { get; } internal string DeploymentName { get; } internal string ServiceName { get; } @@ -25,4 +23,18 @@ return servicePortMap[containerRecipe]; } } + + public class PodInfo + { + public PodInfo(string podName, string podIp, string k8sNodeName) + { + Name = podName; + Ip = podIp; + K8SNodeName = k8sNodeName; + } + + public string Name { get; } + public string Ip { get; } + public string K8SNodeName { get; } + } } diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs index d4edbed..d03633b 100644 --- a/Tests/BasicTests/TwoClientTests.cs +++ b/Tests/BasicTests/TwoClientTests.cs @@ -28,11 +28,10 @@ namespace Tests.BasicTests } [Test] - [Ignore("Requires Location map to be configured for k8s cluster.")] public void TwoClientsTwoLocationsTest() { - var primary = SetupCodexNode(s => s.At(Location.BensLaptop)); - var secondary = SetupCodexNode(s => s.At(Location.BensOldGamingMachine)); + var primary = SetupCodexNode(s => s.At(Location.One)); + var secondary = SetupCodexNode(s => s.At(Location.Two)); PerformTwoClientTest(primary, secondary); }