Merge branch 'feature/better-start-waiting'

Framework/KubernetesWorkflow/K8sController.cs

@@ -43,6 +43,11 @@ namespace KubernetesWorkflow
             return new StartResult(cluster, containerRecipes, deployment, internalService, externalService);
         }
 
+        public void WaitUntilOnline(RunningContainer container)
+        {
+            WaitUntilDeploymentOnline(container.Recipe.Name);
+        }
+
         public PodInfo GetPodInfo(RunningDeployment deployment)
         {
             var pod = GetPodForDeployment(deployment);
@@ -372,7 +377,6 @@ namespace KubernetesWorkflow
             };
 
             client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
-            WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
 
             var name = deploymentSpec.Metadata.Name;
             return new RunningDeployment(name, podLabel);

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -9,8 +9,8 @@ namespace KubernetesWorkflow
     public interface IStartupWorkflow
     {
         IKnownLocations GetAvailableLocations();
-        RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
-        RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
         PodInfo GetPodInfo(RunningContainer container);
         PodInfo GetPodInfo(RunningContainers containers);
         CrashWatcher CreateCrashWatcher(RunningContainer container);
@@ -45,12 +45,12 @@ namespace KubernetesWorkflow
             return locationProvider.GetAvailableLocations();
         }
 
-        public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return Start(numberOfContainers, KnownLocations.UnspecifiedLocation, recipeFactory, startupConfig);
         }
 
-        public RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
+        public FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             return K8s(controller =>
             {
@@ -67,7 +67,18 @@ namespace KubernetesWorkflow
                 {
                     componentFactory.Update(controller);
                 }
-                return rc;
+                return new FutureContainers(rc, this);
+            });
+        }
+
+        public void WaitUntilOnline(RunningContainers rc)
+        {
+            K8s(controller =>
+            {
+                foreach (var c in rc.Containers)
+                {
+                    controller.WaitUntilOnline(c);
+                }
             });
         }
 

Framework/KubernetesWorkflow/Types/FutureContainers.cs

@@ -0,0 +1,20 @@
+namespace KubernetesWorkflow.Types
+{
+    public class FutureContainers
+    {
+        private readonly RunningContainers runningContainers;
+        private readonly StartupWorkflow workflow;
+
+        public FutureContainers(RunningContainers runningContainers, StartupWorkflow workflow)
+        {
+            this.runningContainers = runningContainers;
+            this.workflow = workflow;
+        }
+
+        public RunningContainers WaitForOnline()
+        {
+            workflow.WaitUntilOnline(runningContainers);
+            return runningContainers;
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -24,7 +24,7 @@ namespace CodexContractsPlugin
             var startupConfig = CreateStartupConfig(gethNode);
             startupConfig.NameOverride = "codex-contracts";
 
-            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig).WaitForOnline();
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure.");
             var container = containers.Containers[0];
 

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs

@@ -46,14 +46,14 @@ namespace CodexDiscordBotPlugin
             var startupConfig = new StartupConfig();
             startupConfig.NameOverride = config.Name;
             startupConfig.Add(config);
-            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig);
+            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline();
         }
 
         private RunningContainers StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config)
         {
             var startupConfig = new StartupConfig();
             startupConfig.Add(config);
-            return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig);
+            return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline();
         }
     }
 }

ProjectPlugins/CodexPlugin/CodexStarter.cs

@@ -87,13 +87,16 @@ namespace CodexPlugin
 
         private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
         {
-            var result = new List<RunningContainers>();
+            var futureContainers = new List<FutureContainers>();
             for (var i = 0; i < numberOfNodes; i++)
             {
                 var workflow = pluginTools.CreateWorkflow();
-                result.Add(workflow.Start(1, location, recipe, startupConfig));
+                futureContainers.Add(workflow.Start(1, location, recipe, startupConfig));
             }
-            return result.ToArray();
+
+            return futureContainers
+                .Select(f => f.WaitForOnline())
+                .ToArray();
         }
 
         private PodInfo GetPodInfo(RunningContainers rc)

ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs

@@ -30,7 +30,7 @@ namespace DeployAndRunPlugin
             startupConfig.Add(config);
 
             var location = workflow.GetAvailableLocations().Get("fixed-s-4vcpu-16gb-amd-yz8rd");
-            var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig).WaitForOnline();
             return containers.Containers.Single();
         }
     }

ProjectPlugins/GethPlugin/GethStarter.cs

@@ -21,7 +21,7 @@ namespace GethPlugin
             startupConfig.NameOverride = gethStartupConfig.NameOverride;
 
             var workflow = tools.CreateWorkflow();
-            var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig).WaitForOnline();
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
             var container = containers.Containers[0];
 

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -25,7 +25,7 @@ namespace MetricsPlugin
             startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
 
             var workflow = tools.CreateWorkflow();
-            var runningContainers = workflow.Start(1, recipe, startupConfig);
+            var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 
             Log("Metrics server started.");

Tests/CodexTests/BasicTests/MarketplaceTests.cs

@@ -20,22 +20,23 @@ namespace CodexTests.BasicTests
             var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
             var contracts = Ci.StartCodexContracts(geth);
             
-            var numberOfHosts = 3;
+            var numberOfHosts = 5;
-            for (var i = 0; i < numberOfHosts; i++)
+            var hosts = AddCodex(numberOfHosts, s => s
-            {
+                .WithName("Host")
-                var host = AddCodex(s => s
+                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
-                    .WithName("Host")
+                {
-                    .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)
+                    ContractClock = CodexLogLevel.Trace,
-                    {
+                })
-                        ContractClock = CodexLogLevel.Trace,
+                .WithStorageQuota(11.GB())
-                    })
+                .EnableMarketplace(geth, contracts, m => m
-                    .WithStorageQuota(11.GB())
+                    .WithInitial(10.Eth(), hostInitialBalance)
-                    .EnableMarketplace(geth, contracts, m => m
+                    .AsStorageNode()
-                        .WithInitial(10.Eth(), hostInitialBalance)
+                    .AsValidator()));
-                        .AsStorageNode()
-                        .AsValidator()));
 
-                AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance));
+            var expectedHostBalance = (numberOfHosts * hostInitialBalance.Amount).TestTokens();
+            foreach (var host in hosts)
+            {
+                AssertBalance(contracts, host, Is.EqualTo(expectedHostBalance));
 
                 var availability = new StorageAvailability(
                     totalSpace: 10.GB(),