From 8a6c5e95ee84569af2cb09c65f68bfc1333e6f23 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 7 Aug 2023 10:44:48 +0200 Subject: [PATCH] Makes sure codex nodes are always started in their own pods. --- CodexNetDeployer/Deployer.cs | 2 +- DistTestCore/CodexNodeGroup.cs | 6 ++--- DistTestCore/CodexStarter.cs | 32 +++++++++++++++---------- DistTestCore/OnlineCodexNode.cs | 7 +----- DistTestCore/PrometheusStarter.cs | 4 ++-- KubernetesWorkflow/RunningContainers.cs | 13 ++++++++++ Tests/BasicTests/ExampleTests.cs | 4 ---- 7 files changed, 39 insertions(+), 29 deletions(-) diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index e7180a6..4bfa25d 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -88,7 +88,7 @@ namespace CodexNetDeployer if (!setup.MetricsEnabled) return null; Log("Starting metrics service..."); - var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray()); + var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) }; return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single(); } diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 19c4c8c..7759ea7 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -14,12 +14,12 @@ namespace DistTestCore { private readonly TestLifecycle lifecycle; - public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory) + public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory) { this.lifecycle = lifecycle; Setup = setup; Containers = containers; - Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray(); + Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray(); Version = new CodexDebugVersionResponse(); } @@ -45,7 +45,7 @@ namespace DistTestCore } public CodexSetup Setup { get; private set; } - public RunningContainers Containers { get; private set; } + public RunningContainers[] Containers { get; private set; } public OnlineCodexNode[] Nodes { get; private set; } public CodexDebugVersionResponse Version { get; private set; } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 4c40a06..690ee94 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -32,12 +32,12 @@ namespace DistTestCore var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); lifecycle.SetCodexVersion(group.Version); - var podInfo = group.Containers.RunningPod.PodInfo; + var nl = Environment.NewLine; + var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}")); LogEnd($"Started {codexSetup.NumberOfNodes} nodes " + - $"of image '{containers.Containers.First().Recipe.Image}' " + - $"and version '{group.Version}' " + - $"at location '{podInfo.K8SNodeName}'={podInfo.Ip}. " + - $"They are: {group.Describe()}"); + $"of image '{containers.Containers().First().Recipe.Image}' " + + $"and version '{group.Version}'{nl}" + + podInfos); LogSeparator(); return group; @@ -47,7 +47,7 @@ namespace DistTestCore { LogStart($"Stopping {group.Describe()}..."); var workflow = CreateWorkflow(); - workflow.Stop(group.Containers); + foreach (var c in group.Containers) workflow.Stop(c); RunningGroups.Remove(group); LogEnd("Stopped."); } @@ -66,7 +66,7 @@ namespace DistTestCore workflow.DownloadContainerLog(container, logHandler); } - private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers containers) + private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers) { if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); @@ -83,13 +83,19 @@ namespace DistTestCore return startupConfig; } - private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location) + private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location) { - var workflow = CreateWorkflow(); - return workflow.Start(numberOfNodes, location, new CodexContainerRecipe(), startupConfig); + var result = new List(); + var recipe = new CodexContainerRecipe(); + for (var i = 0; i < numberOfNodes; i++) + { + var workflow = CreateWorkflow(); + result.Add(workflow.Start(1, location, recipe, startupConfig)); + } + return result.ToArray(); } - private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory) + private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory) { var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); RunningGroups.Add(group); @@ -107,10 +113,10 @@ namespace DistTestCore return group; } - private void CodexNodesNotOnline(RunningContainers runningContainers) + private void CodexNodesNotOnline(RunningContainers[] runningContainers) { Log("Codex nodes failed to start"); - foreach (var container in runningContainers.Containers) lifecycle.DownloadLog(container); + foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container); } private StartupWorkflow CreateWorkflow() diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 5a7658c..d547dcc 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -135,14 +135,9 @@ namespace DistTestCore var multiAddress = peerInfo.addrs.First(); // Todo: Is there a case where First address in list is not the way? - if (Group == peer.Group) - { - return multiAddress; - } - // The peer we want to connect is in a different pod. // We must replace the default IP with the pod IP in the multiAddress. - return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip); + return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip); } private void DownloadToFile(string contentId, TestFile file) diff --git a/DistTestCore/PrometheusStarter.cs b/DistTestCore/PrometheusStarter.cs index a0fc9a4..6ef2e57 100644 --- a/DistTestCore/PrometheusStarter.cs +++ b/DistTestCore/PrometheusStarter.cs @@ -12,11 +12,11 @@ namespace DistTestCore { } - public RunningContainers CollectMetricsFor(RunningContainers containers) + public RunningContainers CollectMetricsFor(RunningContainers[] containers) { LogStart($"Starting metrics server for {containers.Describe()}"); var startupConfig = new StartupConfig(); - startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers))); + startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers()))); var workflow = workflowCreator.CreateWorkflow(); var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs index bbfe360..333e4cd 100644 --- a/KubernetesWorkflow/RunningContainers.cs +++ b/KubernetesWorkflow/RunningContainers.cs @@ -40,4 +40,17 @@ namespace KubernetesWorkflow public Address ClusterExternalAddress { get; } public Address ClusterInternalAddress { get; } } + + public static class RunningContainersExtensions + { + public static RunningContainer[] Containers(this RunningContainers[] runningContainers) + { + return runningContainers.SelectMany(c => c.Containers).ToArray(); + } + + public static string Describe(this RunningContainers[] runningContainers) + { + return string.Join(",", runningContainers.Select(c => c.Describe())); + } + } } diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 420b6fa..58d4cbc 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -1,9 +1,5 @@ using DistTestCore; -using DistTestCore.Codex; -using DistTestCore.Marketplace; -using Newtonsoft.Json; using NUnit.Framework; -using Utils; namespace Tests.BasicTests {