From 3a8bb760efcca3bfe50ea63094d4b8cb9b07a99f Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 19 Oct 2023 11:08:30 +0200 Subject: [PATCH] Adding support for multiple exposed container ports --- .../ContainerRecipeFactory.cs | 5 --- .../KubernetesWorkflow/RunnerLocationUtils.cs | 22 ++++++--- .../KubernetesWorkflow/RunningContainers.cs | 33 +++++++++----- .../KubernetesWorkflow/StartupWorkflow.cs | 45 ++++++++++--------- .../CodexPlugin/CodexContainerRecipe.cs | 4 +- 5 files changed, 64 insertions(+), 45 deletions(-) diff --git a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs index 6fb50a57..8d19398f 100644 --- a/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs +++ b/Framework/KubernetesWorkflow/ContainerRecipeFactory.cs @@ -132,11 +132,6 @@ namespace KubernetesWorkflow private Port AddExposedPort(Port port) { - if (exposedPorts.Any()) - { - throw new NotImplementedException("Current implementation only support 1 exposed port per container recipe. " + - $"Methods for determining container addresses in {nameof(StartupWorkflow)} currently rely on this constraint."); - } exposedPorts.Add(port); return port; } diff --git a/Framework/KubernetesWorkflow/RunnerLocationUtils.cs b/Framework/KubernetesWorkflow/RunnerLocationUtils.cs index 055aa153..86984f8d 100644 --- a/Framework/KubernetesWorkflow/RunnerLocationUtils.cs +++ b/Framework/KubernetesWorkflow/RunnerLocationUtils.cs @@ -16,18 +16,26 @@ namespace KubernetesWorkflow internal static RunnerLocation DetermineRunnerLocation(RunningContainer container) { if (knownLocation != null) return knownLocation.Value; + knownLocation = PingForLocation(container); + return knownLocation.Value; + } + private static RunnerLocation PingForLocation(RunningContainer container) + { if (PingHost(container.Pod.PodInfo.Ip)) { - knownLocation = RunnerLocation.InternalToCluster; - } - else if (PingHost(Format(container.ClusterExternalAddress))) - { - knownLocation = RunnerLocation.ExternalToCluster; + return RunnerLocation.InternalToCluster; } - if (knownLocation == null) throw new Exception("Unable to determine location relative to kubernetes cluster."); - return knownLocation.Value; + foreach (var port in container.ContainerPorts) + { + if (PingHost(Format(port.ExternalAddress))) + { + return RunnerLocation.ExternalToCluster; + } + } + + throw new Exception("Unable to determine location relative to kubernetes cluster."); } private static string Format(Address host) diff --git a/Framework/KubernetesWorkflow/RunningContainers.cs b/Framework/KubernetesWorkflow/RunningContainers.cs index cfcae4da..f515fb9e 100644 --- a/Framework/KubernetesWorkflow/RunningContainers.cs +++ b/Framework/KubernetesWorkflow/RunningContainers.cs @@ -24,37 +24,50 @@ namespace KubernetesWorkflow public class RunningContainer { - public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, Address clusterExternalAddress, Address clusterInternalAddress) + public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, ContainerPort[] containerPorts) { Pod = pod; Recipe = recipe; ServicePorts = servicePorts; Name = name; - ClusterExternalAddress = clusterExternalAddress; - ClusterInternalAddress = clusterInternalAddress; + ContainerPorts = containerPorts; } public string Name { get; } public RunningPod Pod { get; } public ContainerRecipe Recipe { get; } public Port[] ServicePorts { get; } - public Address ClusterExternalAddress { get; } - public Address ClusterInternalAddress { get; } + public ContainerPort[] ContainerPorts { get; } [JsonIgnore] public Address Address { get { - if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster) - { - return ClusterInternalAddress; - } - return ClusterExternalAddress; + throw new Exception("a"); + //if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster) + //{ + // return ClusterInternalAddress; + //} + //return ClusterExternalAddress; } } } + public class ContainerPort + { + public ContainerPort(Port port, Address externalAddress, Address internalAddress) + { + Port = port; + ExternalAddress = externalAddress; + InternalAddress = internalAddress; + } + + public Port Port { get; } + public Address ExternalAddress { get; } + public Address InternalAddress { get; } + } + public static class RunningContainersExtensions { public static RunningContainer[] Containers(this RunningContainers[] runningContainers) diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs index bda8a9b9..e50e39e3 100644 --- a/Framework/KubernetesWorkflow/StartupWorkflow.cs +++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs @@ -118,8 +118,7 @@ namespace KubernetesWorkflow var name = GetContainerName(r, startupConfig); return new RunningContainer(runningPod, r, servicePorts, name, - GetContainerExternalAddress(runningPod, servicePorts), - GetContainerInternalAddress(r)); + CreateContainerPorts(runningPod, r, servicePorts)); }).ToArray(); } @@ -137,35 +136,39 @@ namespace KubernetesWorkflow } } - private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts) + private ContainerPort[] CreateContainerPorts(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts) { - return new Address( - pod.Cluster.HostAddress, - GetServicePort(servicePorts)); + var result = new List(); + foreach (var exposedPort in recipe.ExposedPorts) + { + result.Add(new ContainerPort( + exposedPort, + GetContainerExternalAddress(pod, servicePorts, exposedPort), + GetContainerInternalAddress(exposedPort))); + } + + return result.ToArray(); } - private Address GetContainerInternalAddress(ContainerRecipe recipe) + private static Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts, Port exposedPort) + { + var servicePort = servicePorts.Single(p => p.Tag == exposedPort.Tag); + + return new Address( + pod.Cluster.HostAddress, + servicePort.Number); + } + + private Address GetContainerInternalAddress(Port exposedPort) { var serviceName = "service-" + numberSource.WorkflowNumber; - var port = GetInternalPort(recipe); + var port = exposedPort.Number; return new Address( $"http://{serviceName}.{k8sNamespace}.svc.cluster.local", port); } - - private static int GetServicePort(Port[] servicePorts) - { - if (servicePorts.Any()) return servicePorts.First().Number; - return 0; - } - - private static int GetInternalPort(ContainerRecipe recipe) - { - if (recipe.ExposedPorts.Any()) return recipe.ExposedPorts.First().Number; - return 0; - } - + private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { log.Debug(); diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index ebef9932..905041aa 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -34,13 +34,13 @@ namespace CodexPlugin AddEnvVar("CODEX_DATA_DIR", dataDir); AddVolume($"codex/{dataDir}", GetVolumeCapacity(config)); - AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag); + AddExposedPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag); AddEnvVar("CODEX_LOG_LEVEL", config.LogLevelWithTopics()); // This makes the node announce itself to its local (pod) IP address. AddEnvVar("NAT_IP_AUTO", "true"); - var listenPort = AddInternalPort(); + var listenPort = AddExposedPort(); AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); if (!string.IsNullOrEmpty(config.BootstrapSpr))