From b47b5960629379268c8996a53f1981d5ed222912 Mon Sep 17 00:00:00 2001
From: benbierens <thatbenbierens@gmail.com>
Date: Tue, 14 Nov 2023 10:49:14 +0100
Subject: [PATCH] Fetches used external ports in order to guarantee no
 collisions.

---
 Framework/KubernetesWorkflow/K8sController.cs | 25 +++++++++++++++++++
 .../Recipe/ContainerRecipeFactory.cs          |  2 +-
 .../Recipe/RecipeComponentFactory.cs          | 24 ++++++++++++++----
 .../KubernetesWorkflow/StartupWorkflow.cs     |  7 ++++++
 4 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs
index e0ca025..5b657d2 100644
--- a/Framework/KubernetesWorkflow/K8sController.cs
+++ b/Framework/KubernetesWorkflow/K8sController.cs
@@ -84,6 +84,31 @@ namespace KubernetesWorkflow
             return result;
         }
 
+        public int[] GetUsedExternalPorts()
+        {
+            return client.Run(c =>
+            {
+                var result = new List<int>();
+
+                var services = c.ListServiceForAllNamespaces();
+                var nodePorts = services.Items.Where(s => s.Spec.Type == "NodePort").ToArray();
+                if (!nodePorts.Any()) return result.ToArray();
+
+                foreach (var service in nodePorts)
+                {
+                    foreach (var port in service.Spec.Ports)
+                    {
+                        if (port.NodePort.HasValue)
+                        {
+                            result.Add(port.NodePort.Value);
+                        }
+                    }
+                }
+
+                return result.ToArray();
+            });
+        }
+
         public void DeleteAllNamespacesStartingWith(string prefix)
         {
             log.Debug();
diff --git a/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs b/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs
index c10d4a1..0a988de 100644
--- a/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs
+++ b/Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs
@@ -57,7 +57,7 @@ namespace KubernetesWorkflow.Recipe
 
         protected Port AddExposedPort(int number, string tag, PortProtocol protocol = PortProtocol.TCP)
         {
-            return AddExposedPort(factory.CreatePort(number, tag, protocol));
+            return AddExposedPort(factory.CreateExternalPort(number, tag, protocol));
         }
 
         protected Port AddInternalPort(string tag = "", PortProtocol protocol = PortProtocol.TCP)
diff --git a/Framework/KubernetesWorkflow/Recipe/RecipeComponentFactory.cs b/Framework/KubernetesWorkflow/Recipe/RecipeComponentFactory.cs
index 8b4ef7f..3935385 100644
--- a/Framework/KubernetesWorkflow/Recipe/RecipeComponentFactory.cs
+++ b/Framework/KubernetesWorkflow/Recipe/RecipeComponentFactory.cs
@@ -5,12 +5,13 @@ namespace KubernetesWorkflow.Recipe
 {
     public class RecipeComponentFactory
     {
-        private NumberSource internalNumberSource = new NumberSource(8080);
-        private static NumberSource externalNumberSource = new NumberSource(30000);
+        private readonly NumberSource internalNumberSource = new NumberSource(8080);
+        private static readonly NumberSource externalNumberSource = new NumberSource(30000);
+        private static int[] usedExternalPorts = Array.Empty<int>();
 
-        public Port CreatePort(int number, string tag, PortProtocol protocol)
+        public void Update(K8sController controller)
         {
-            return new Port(number, tag, protocol);
+            usedExternalPorts = controller.GetUsedExternalPorts();
         }
 
         public Port CreateInternalPort(string tag, PortProtocol protocol)
@@ -18,9 +19,22 @@ namespace KubernetesWorkflow.Recipe
             return new Port(internalNumberSource.GetNextNumber(), tag, protocol);
         }
 
+        public Port CreateExternalPort(int number, string tag, PortProtocol protocol)
+        {
+            if (usedExternalPorts.Contains(number)) throw new Exception($"External port number {number} is already in use by the cluster.");
+            return new Port(number, tag, protocol);
+        }
+
         public Port CreateExternalPort(string tag, PortProtocol protocol)
         {
-            return new Port(externalNumberSource.GetNextNumber(), tag, protocol);
+            while (true)
+            {
+                var number = externalNumberSource.GetNextNumber();
+                if (!usedExternalPorts.Contains(number))
+                {
+                    return new Port(number, tag, protocol);
+                }
+            }
         }
 
         public EnvVar CreateEnvVar(string name, int value)
diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs
index 49bc764..15bc3e3 100644
--- a/Framework/KubernetesWorkflow/StartupWorkflow.cs
+++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs
@@ -54,12 +54,19 @@ namespace KubernetesWorkflow
         {
             return K8s(controller =>
             {
+                componentFactory.Update(controller);
+
                 var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
                 var rc = new RunningContainers(startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
+
+                if (startResult.ExternalService != null)
+                {
+                    componentFactory.Update(controller);
+                }
                 return rc;
             });
         }