diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index b14267d..80b15d1 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -43,6 +43,11 @@ namespace KubernetesWorkflow return new StartResult(cluster, containerRecipes, deployment, internalService, externalService); } + public void WaitUntilOnline(RunningContainer container) + { + WaitUntilDeploymentOnline(container.Recipe.Name); + } + public PodInfo GetPodInfo(RunningDeployment deployment) { var pod = GetPodForDeployment(deployment); @@ -372,7 +377,6 @@ namespace KubernetesWorkflow }; client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace)); - WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name); var name = deploymentSpec.Metadata.Name; return new RunningDeployment(name, podLabel); diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs index 4d1a2f6..b8f7d77 100644 --- a/Framework/KubernetesWorkflow/StartupWorkflow.cs +++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs @@ -9,8 +9,8 @@ namespace KubernetesWorkflow public interface IStartupWorkflow { IKnownLocations GetAvailableLocations(); - RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig); - RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig); + FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig); + FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig); PodInfo GetPodInfo(RunningContainer container); PodInfo GetPodInfo(RunningContainers containers); CrashWatcher CreateCrashWatcher(RunningContainer container); @@ -45,12 +45,12 @@ namespace KubernetesWorkflow return locationProvider.GetAvailableLocations(); } - public RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) + public FutureContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { return Start(numberOfContainers, KnownLocations.UnspecifiedLocation, recipeFactory, startupConfig); } - public RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) + public FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { return K8s(controller => { @@ -67,7 +67,18 @@ namespace KubernetesWorkflow { componentFactory.Update(controller); } - return rc; + return new FutureContainers(rc, this); + }); + } + + public void WaitUntilOnline(RunningContainers rc) + { + K8s(controller => + { + foreach (var c in rc.Containers) + { + controller.WaitUntilOnline(c); + } }); } diff --git a/Framework/KubernetesWorkflow/Types/FutureContainers.cs b/Framework/KubernetesWorkflow/Types/FutureContainers.cs new file mode 100644 index 0000000..262eac4 --- /dev/null +++ b/Framework/KubernetesWorkflow/Types/FutureContainers.cs @@ -0,0 +1,20 @@ +namespace KubernetesWorkflow.Types +{ + public class FutureContainers + { + private readonly RunningContainers runningContainers; + private readonly StartupWorkflow workflow; + + public FutureContainers(RunningContainers runningContainers, StartupWorkflow workflow) + { + this.runningContainers = runningContainers; + this.workflow = workflow; + } + + public RunningContainers WaitForOnline() + { + workflow.WaitUntilOnline(runningContainers); + return runningContainers; + } + } +} diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs index a6f6641..0137a17 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs @@ -24,7 +24,7 @@ namespace CodexContractsPlugin var startupConfig = CreateStartupConfig(gethNode); startupConfig.NameOverride = "codex-contracts"; - var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig); + var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig).WaitForOnline(); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure."); var container = containers.Containers[0]; diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs index cad50c1..fd082cf 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs @@ -46,14 +46,14 @@ namespace CodexDiscordBotPlugin var startupConfig = new StartupConfig(); startupConfig.NameOverride = config.Name; startupConfig.Add(config); - return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig); + return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline(); } private RunningContainers StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config) { var startupConfig = new StartupConfig(); startupConfig.Add(config); - return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig); + return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline(); } } } diff --git a/ProjectPlugins/CodexPlugin/CodexStarter.cs b/ProjectPlugins/CodexPlugin/CodexStarter.cs index 05db6bd..6fcc4bd 100644 --- a/ProjectPlugins/CodexPlugin/CodexStarter.cs +++ b/ProjectPlugins/CodexPlugin/CodexStarter.cs @@ -87,13 +87,16 @@ namespace CodexPlugin private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location) { - var result = new List(); + var futureContainers = new List(); for (var i = 0; i < numberOfNodes; i++) { var workflow = pluginTools.CreateWorkflow(); - result.Add(workflow.Start(1, location, recipe, startupConfig)); + futureContainers.Add(workflow.Start(1, location, recipe, startupConfig)); } - return result.ToArray(); + + return futureContainers + .Select(f => f.WaitForOnline()) + .ToArray(); } private PodInfo GetPodInfo(RunningContainers rc) diff --git a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs index 070b9fe..c290c0d 100644 --- a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs +++ b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs @@ -30,7 +30,7 @@ namespace DeployAndRunPlugin startupConfig.Add(config); var location = workflow.GetAvailableLocations().Get("fixed-s-4vcpu-16gb-amd-yz8rd"); - var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig); + var containers = workflow.Start(1, location, new DeployAndRunContainerRecipe(), startupConfig).WaitForOnline(); return containers.Containers.Single(); } } diff --git a/ProjectPlugins/GethPlugin/GethStarter.cs b/ProjectPlugins/GethPlugin/GethStarter.cs index c5ad801..edfba4e 100644 --- a/ProjectPlugins/GethPlugin/GethStarter.cs +++ b/ProjectPlugins/GethPlugin/GethStarter.cs @@ -21,7 +21,7 @@ namespace GethPlugin startupConfig.NameOverride = gethStartupConfig.NameOverride; var workflow = tools.CreateWorkflow(); - var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig); + var containers = workflow.Start(1, new GethContainerRecipe(), startupConfig).WaitForOnline(); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure."); var container = containers.Containers[0]; diff --git a/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs b/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs index 53f784e..f63f97d 100644 --- a/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs +++ b/ProjectPlugins/MetricsPlugin/PrometheusStarter.cs @@ -25,7 +25,7 @@ namespace MetricsPlugin startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets))); var workflow = tools.CreateWorkflow(); - var runningContainers = workflow.Start(1, recipe, startupConfig); + var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline(); if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created."); Log("Metrics server started."); diff --git a/Tests/CodexTests/BasicTests/MarketplaceTests.cs b/Tests/CodexTests/BasicTests/MarketplaceTests.cs index 7474696..d5befe0 100644 --- a/Tests/CodexTests/BasicTests/MarketplaceTests.cs +++ b/Tests/CodexTests/BasicTests/MarketplaceTests.cs @@ -19,23 +19,24 @@ namespace CodexTests.BasicTests var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth")); var contracts = Ci.StartCodexContracts(geth); + + var numberOfHosts = 5; + var hosts = AddCodex(numberOfHosts, s => s + .WithName("Host") + .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) + { + ContractClock = CodexLogLevel.Trace, + }) + .WithStorageQuota(11.GB()) + .EnableMarketplace(geth, contracts, m => m + .WithInitial(10.Eth(), hostInitialBalance) + .AsStorageNode() + .AsValidator())); - var numberOfHosts = 3; - for (var i = 0; i < numberOfHosts; i++) + var expectedHostBalance = (numberOfHosts * hostInitialBalance.Amount).TestTokens(); + foreach (var host in hosts) { - var host = AddCodex(s => s - .WithName("Host") - .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) - { - ContractClock = CodexLogLevel.Trace, - }) - .WithStorageQuota(11.GB()) - .EnableMarketplace(geth, contracts, m => m - .WithInitial(10.Eth(), hostInitialBalance) - .AsStorageNode() - .AsValidator())); - - AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance)); + AssertBalance(contracts, host, Is.EqualTo(expectedHostBalance)); var availability = new StorageAvailability( totalSpace: 10.GB(),