2023-04-12 14:12:04 +00:00
|
|
|
|
using DistTestCore.Codex;
|
2023-04-30 08:56:19 +00:00
|
|
|
|
using DistTestCore.Marketplace;
|
2023-07-11 10:21:48 +00:00
|
|
|
|
using DistTestCore.Metrics;
|
2023-04-12 14:12:04 +00:00
|
|
|
|
using KubernetesWorkflow;
|
2023-05-11 10:44:53 +00:00
|
|
|
|
using Logging;
|
2023-04-12 14:06:04 +00:00
|
|
|
|
|
|
|
|
|
namespace DistTestCore
|
|
|
|
|
{
|
2023-04-18 11:45:48 +00:00
|
|
|
|
public class CodexStarter : BaseStarter
|
2023-04-12 14:06:04 +00:00
|
|
|
|
{
|
2023-08-10 08:58:18 +00:00
|
|
|
|
public CodexStarter(TestLifecycle lifecycle)
|
|
|
|
|
: base(lifecycle)
|
2023-04-12 14:06:04 +00:00
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
2023-04-13 09:53:54 +00:00
|
|
|
|
public List<CodexNodeGroup> RunningGroups { get; } = new List<CodexNodeGroup>();
|
|
|
|
|
|
2023-04-13 07:33:10 +00:00
|
|
|
|
public ICodexNodeGroup BringOnline(CodexSetup codexSetup)
|
2023-04-12 14:06:04 +00:00
|
|
|
|
{
|
2023-04-19 07:19:06 +00:00
|
|
|
|
LogSeparator();
|
2023-04-18 11:45:48 +00:00
|
|
|
|
LogStart($"Starting {codexSetup.Describe()}...");
|
2023-04-14 08:51:35 +00:00
|
|
|
|
var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
|
|
|
|
|
|
2023-04-30 08:56:19 +00:00
|
|
|
|
var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
|
2023-07-31 09:51:29 +00:00
|
|
|
|
|
2023-04-14 08:51:35 +00:00
|
|
|
|
var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
|
2023-04-13 09:53:54 +00:00
|
|
|
|
|
2023-07-11 10:21:48 +00:00
|
|
|
|
var metricAccessFactory = CollectMetrics(codexSetup, containers);
|
2023-04-12 14:06:04 +00:00
|
|
|
|
|
2023-04-14 08:51:35 +00:00
|
|
|
|
var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
|
2023-04-12 14:12:04 +00:00
|
|
|
|
|
2023-04-13 12:36:17 +00:00
|
|
|
|
var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
|
2023-07-31 09:51:29 +00:00
|
|
|
|
lifecycle.SetCodexVersion(group.Version);
|
|
|
|
|
|
2023-08-07 08:44:48 +00:00
|
|
|
|
var nl = Environment.NewLine;
|
|
|
|
|
var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
|
2023-07-31 09:51:29 +00:00
|
|
|
|
LogEnd($"Started {codexSetup.NumberOfNodes} nodes " +
|
2023-08-07 08:44:48 +00:00
|
|
|
|
$"of image '{containers.Containers().First().Recipe.Image}' " +
|
|
|
|
|
$"and version '{group.Version}'{nl}" +
|
|
|
|
|
podInfos);
|
2023-04-19 07:19:06 +00:00
|
|
|
|
LogSeparator();
|
2023-07-31 09:51:29 +00:00
|
|
|
|
|
2023-04-13 09:53:54 +00:00
|
|
|
|
return group;
|
2023-04-12 14:06:04 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-04-13 09:53:54 +00:00
|
|
|
|
public void BringOffline(CodexNodeGroup group)
|
2023-04-13 09:07:36 +00:00
|
|
|
|
{
|
2023-04-18 11:45:48 +00:00
|
|
|
|
LogStart($"Stopping {group.Describe()}...");
|
2023-04-13 09:07:36 +00:00
|
|
|
|
var workflow = CreateWorkflow();
|
2023-08-07 08:44:48 +00:00
|
|
|
|
foreach (var c in group.Containers) workflow.Stop(c);
|
2023-04-13 09:53:54 +00:00
|
|
|
|
RunningGroups.Remove(group);
|
2023-04-18 11:45:48 +00:00
|
|
|
|
LogEnd("Stopped.");
|
2023-04-13 09:07:36 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-04-12 14:06:04 +00:00
|
|
|
|
public void DeleteAllResources()
|
|
|
|
|
{
|
2023-04-13 09:07:36 +00:00
|
|
|
|
var workflow = CreateWorkflow();
|
2023-05-03 12:18:37 +00:00
|
|
|
|
workflow.DeleteTestResources();
|
2023-04-13 09:53:54 +00:00
|
|
|
|
|
|
|
|
|
RunningGroups.Clear();
|
2023-04-12 14:06:04 +00:00
|
|
|
|
}
|
2023-04-13 09:07:36 +00:00
|
|
|
|
|
2023-04-13 09:30:19 +00:00
|
|
|
|
public void DownloadLog(RunningContainer container, ILogHandler logHandler)
|
|
|
|
|
{
|
|
|
|
|
var workflow = CreateWorkflow();
|
|
|
|
|
workflow.DownloadContainerLog(container, logHandler);
|
|
|
|
|
}
|
2023-04-30 08:56:19 +00:00
|
|
|
|
|
2023-08-07 08:44:48 +00:00
|
|
|
|
private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
|
2023-07-11 10:21:48 +00:00
|
|
|
|
{
|
2023-08-13 07:07:23 +00:00
|
|
|
|
if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory();
|
2023-07-11 10:21:48 +00:00
|
|
|
|
|
|
|
|
|
var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
|
2023-08-13 07:07:23 +00:00
|
|
|
|
|
|
|
|
|
if (codexSetup.MetricsMode == MetricsMode.Dashboard)
|
|
|
|
|
{
|
2023-08-14 08:26:04 +00:00
|
|
|
|
lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup);
|
2023-08-13 07:07:23 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-07-11 10:21:48 +00:00
|
|
|
|
return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
|
|
|
|
|
}
|
|
|
|
|
|
2023-04-30 08:56:19 +00:00
|
|
|
|
private StartupConfig CreateStartupConfig(GethStartResult gethStartResult, CodexSetup codexSetup)
|
|
|
|
|
{
|
|
|
|
|
var startupConfig = new StartupConfig();
|
|
|
|
|
startupConfig.NameOverride = codexSetup.NameOverride;
|
|
|
|
|
startupConfig.Add(codexSetup);
|
|
|
|
|
startupConfig.Add(gethStartResult);
|
|
|
|
|
return startupConfig;
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-07 08:44:48 +00:00
|
|
|
|
private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
|
2023-04-13 12:36:17 +00:00
|
|
|
|
{
|
2023-08-07 08:44:48 +00:00
|
|
|
|
var result = new List<RunningContainers>();
|
|
|
|
|
var recipe = new CodexContainerRecipe();
|
|
|
|
|
for (var i = 0; i < numberOfNodes; i++)
|
|
|
|
|
{
|
|
|
|
|
var workflow = CreateWorkflow();
|
|
|
|
|
result.Add(workflow.Start(1, location, recipe, startupConfig));
|
|
|
|
|
}
|
|
|
|
|
return result.ToArray();
|
2023-04-13 12:36:17 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-08-07 08:44:48 +00:00
|
|
|
|
private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
|
2023-04-13 12:36:17 +00:00
|
|
|
|
{
|
|
|
|
|
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
|
|
|
|
|
RunningGroups.Add(group);
|
2023-08-04 06:19:03 +00:00
|
|
|
|
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
|
|
|
|
|
}
|
|
|
|
|
catch
|
|
|
|
|
{
|
|
|
|
|
CodexNodesNotOnline(runningContainers);
|
|
|
|
|
throw;
|
|
|
|
|
}
|
|
|
|
|
|
2023-04-13 12:36:17 +00:00
|
|
|
|
return group;
|
|
|
|
|
}
|
2023-04-13 09:30:19 +00:00
|
|
|
|
|
2023-08-07 08:44:48 +00:00
|
|
|
|
private void CodexNodesNotOnline(RunningContainers[] runningContainers)
|
2023-08-04 06:19:03 +00:00
|
|
|
|
{
|
|
|
|
|
Log("Codex nodes failed to start");
|
2023-08-07 08:44:48 +00:00
|
|
|
|
foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
|
2023-08-04 06:19:03 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-04-13 09:07:36 +00:00
|
|
|
|
private StartupWorkflow CreateWorkflow()
|
|
|
|
|
{
|
2023-08-10 08:58:18 +00:00
|
|
|
|
return lifecycle.WorkflowCreator.CreateWorkflow();
|
2023-04-13 09:07:36 +00:00
|
|
|
|
}
|
2023-04-19 07:19:06 +00:00
|
|
|
|
|
|
|
|
|
private void LogSeparator()
|
|
|
|
|
{
|
|
|
|
|
Log("----------------------------------------------------------------------------");
|
|
|
|
|
}
|
2023-04-12 14:06:04 +00:00
|
|
|
|
}
|
|
|
|
|
}
|