Makes sure codex nodes are always started in their own pods.

This commit is contained in:
benbierens 2023-08-07 10:44:48 +02:00
parent 771a952f02
commit 8a6c5e95ee
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
7 changed files with 39 additions and 29 deletions

View File

@ -88,7 +88,7 @@ namespace CodexNetDeployer
if (!setup.MetricsEnabled) return null; if (!setup.MetricsEnabled) return null;
Log("Starting metrics service..."); Log("Starting metrics service...");
var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray()); var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single(); return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
} }

View File

@ -14,12 +14,12 @@ namespace DistTestCore
{ {
private readonly TestLifecycle lifecycle; private readonly TestLifecycle lifecycle;
public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory) public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
{ {
this.lifecycle = lifecycle; this.lifecycle = lifecycle;
Setup = setup; Setup = setup;
Containers = containers; Containers = containers;
Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray(); Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
Version = new CodexDebugVersionResponse(); Version = new CodexDebugVersionResponse();
} }
@ -45,7 +45,7 @@ namespace DistTestCore
} }
public CodexSetup Setup { get; private set; } public CodexSetup Setup { get; private set; }
public RunningContainers Containers { get; private set; } public RunningContainers[] Containers { get; private set; }
public OnlineCodexNode[] Nodes { get; private set; } public OnlineCodexNode[] Nodes { get; private set; }
public CodexDebugVersionResponse Version { get; private set; } public CodexDebugVersionResponse Version { get; private set; }

View File

@ -32,12 +32,12 @@ namespace DistTestCore
var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory); var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
lifecycle.SetCodexVersion(group.Version); lifecycle.SetCodexVersion(group.Version);
var podInfo = group.Containers.RunningPod.PodInfo; var nl = Environment.NewLine;
var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
LogEnd($"Started {codexSetup.NumberOfNodes} nodes " + LogEnd($"Started {codexSetup.NumberOfNodes} nodes " +
$"of image '{containers.Containers.First().Recipe.Image}' " + $"of image '{containers.Containers().First().Recipe.Image}' " +
$"and version '{group.Version}' " + $"and version '{group.Version}'{nl}" +
$"at location '{podInfo.K8SNodeName}'={podInfo.Ip}. " + podInfos);
$"They are: {group.Describe()}");
LogSeparator(); LogSeparator();
return group; return group;
@ -47,7 +47,7 @@ namespace DistTestCore
{ {
LogStart($"Stopping {group.Describe()}..."); LogStart($"Stopping {group.Describe()}...");
var workflow = CreateWorkflow(); var workflow = CreateWorkflow();
workflow.Stop(group.Containers); foreach (var c in group.Containers) workflow.Stop(c);
RunningGroups.Remove(group); RunningGroups.Remove(group);
LogEnd("Stopped."); LogEnd("Stopped.");
} }
@ -66,7 +66,7 @@ namespace DistTestCore
workflow.DownloadContainerLog(container, logHandler); workflow.DownloadContainerLog(container, logHandler);
} }
private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers containers) private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
{ {
if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory(); if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
@ -83,13 +83,19 @@ namespace DistTestCore
return startupConfig; return startupConfig;
} }
private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location) private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
{ {
var workflow = CreateWorkflow(); var result = new List<RunningContainers>();
return workflow.Start(numberOfNodes, location, new CodexContainerRecipe(), startupConfig); var recipe = new CodexContainerRecipe();
for (var i = 0; i < numberOfNodes; i++)
{
var workflow = CreateWorkflow();
result.Add(workflow.Start(1, location, recipe, startupConfig));
}
return result.ToArray();
} }
private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory) private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
{ {
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
RunningGroups.Add(group); RunningGroups.Add(group);
@ -107,10 +113,10 @@ namespace DistTestCore
return group; return group;
} }
private void CodexNodesNotOnline(RunningContainers runningContainers) private void CodexNodesNotOnline(RunningContainers[] runningContainers)
{ {
Log("Codex nodes failed to start"); Log("Codex nodes failed to start");
foreach (var container in runningContainers.Containers) lifecycle.DownloadLog(container); foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
} }
private StartupWorkflow CreateWorkflow() private StartupWorkflow CreateWorkflow()

View File

@ -135,14 +135,9 @@ namespace DistTestCore
var multiAddress = peerInfo.addrs.First(); var multiAddress = peerInfo.addrs.First();
// Todo: Is there a case where First address in list is not the way? // Todo: Is there a case where First address in list is not the way?
if (Group == peer.Group)
{
return multiAddress;
}
// The peer we want to connect is in a different pod. // The peer we want to connect is in a different pod.
// We must replace the default IP with the pod IP in the multiAddress. // We must replace the default IP with the pod IP in the multiAddress.
return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip); return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip);
} }
private void DownloadToFile(string contentId, TestFile file) private void DownloadToFile(string contentId, TestFile file)

View File

@ -12,11 +12,11 @@ namespace DistTestCore
{ {
} }
public RunningContainers CollectMetricsFor(RunningContainers containers) public RunningContainers CollectMetricsFor(RunningContainers[] containers)
{ {
LogStart($"Starting metrics server for {containers.Describe()}"); LogStart($"Starting metrics server for {containers.Describe()}");
var startupConfig = new StartupConfig(); var startupConfig = new StartupConfig();
startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers))); startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers())));
var workflow = workflowCreator.CreateWorkflow(); var workflow = workflowCreator.CreateWorkflow();
var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig); var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);

View File

@ -40,4 +40,17 @@ namespace KubernetesWorkflow
public Address ClusterExternalAddress { get; } public Address ClusterExternalAddress { get; }
public Address ClusterInternalAddress { get; } public Address ClusterInternalAddress { get; }
} }
public static class RunningContainersExtensions
{
public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
{
return runningContainers.SelectMany(c => c.Containers).ToArray();
}
public static string Describe(this RunningContainers[] runningContainers)
{
return string.Join(",", runningContainers.Select(c => c.Describe()));
}
}
} }

View File

@ -1,9 +1,5 @@
using DistTestCore; using DistTestCore;
using DistTestCore.Codex;
using DistTestCore.Marketplace;
using Newtonsoft.Json;
using NUnit.Framework; using NUnit.Framework;
using Utils;
namespace Tests.BasicTests namespace Tests.BasicTests
{ {