2023-09-12 11:32:06 +00:00
|
|
|
|
using Core;
|
2023-09-11 14:57:57 +00:00
|
|
|
|
using KubernetesWorkflow;
|
2023-11-12 09:07:23 +00:00
|
|
|
|
using KubernetesWorkflow.Types;
|
2023-09-11 09:59:33 +00:00
|
|
|
|
using Logging;
|
|
|
|
|
|
|
|
|
|
namespace CodexPlugin
|
|
|
|
|
{
|
2023-09-11 14:57:57 +00:00
|
|
|
|
public class CodexStarter
|
2023-09-11 09:59:33 +00:00
|
|
|
|
{
|
2023-09-12 08:31:55 +00:00
|
|
|
|
private readonly IPluginTools pluginTools;
|
2023-09-13 14:06:05 +00:00
|
|
|
|
private readonly CodexContainerRecipe recipe = new CodexContainerRecipe();
|
|
|
|
|
private CodexDebugVersionResponse? versionResponse;
|
2023-09-11 14:57:57 +00:00
|
|
|
|
|
2023-09-12 13:18:36 +00:00
|
|
|
|
public CodexStarter(IPluginTools pluginTools)
|
2023-09-11 14:57:57 +00:00
|
|
|
|
{
|
2023-09-12 13:18:36 +00:00
|
|
|
|
this.pluginTools = pluginTools;
|
2023-09-11 14:57:57 +00:00
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-11 14:57:57 +00:00
|
|
|
|
public RunningContainers[] BringOnline(CodexSetup codexSetup)
|
2023-09-11 09:59:33 +00:00
|
|
|
|
{
|
2023-09-12 13:18:36 +00:00
|
|
|
|
LogSeparator();
|
|
|
|
|
Log($"Starting {codexSetup.Describe()}...");
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-19 11:58:45 +00:00
|
|
|
|
var startupConfig = CreateStartupConfig(codexSetup);
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-12 13:18:36 +00:00
|
|
|
|
var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-11-06 13:33:47 +00:00
|
|
|
|
foreach (var rc in containers)
|
|
|
|
|
{
|
|
|
|
|
var podInfo = GetPodInfo(rc);
|
|
|
|
|
var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
|
|
|
|
|
Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers().First().Recipe.Image}'. ({podInfos})");
|
|
|
|
|
}
|
2023-09-12 13:18:36 +00:00
|
|
|
|
LogSeparator();
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-12 13:18:36 +00:00
|
|
|
|
return containers;
|
2023-09-11 14:57:57 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-09-19 11:58:45 +00:00
|
|
|
|
public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningContainers[] containers)
|
2023-09-11 14:57:57 +00:00
|
|
|
|
{
|
2023-09-19 11:58:45 +00:00
|
|
|
|
var codexNodeFactory = new CodexNodeFactory(pluginTools);
|
2023-09-11 14:57:57 +00:00
|
|
|
|
|
2023-09-19 11:58:45 +00:00
|
|
|
|
var group = CreateCodexGroup(coreInterface, containers, codexNodeFactory);
|
2023-09-11 14:57:57 +00:00
|
|
|
|
|
2023-09-12 13:18:36 +00:00
|
|
|
|
Log($"Codex version: {group.Version}");
|
2023-09-13 14:06:05 +00:00
|
|
|
|
versionResponse = group.Version;
|
2023-09-12 13:18:36 +00:00
|
|
|
|
|
|
|
|
|
return group;
|
2023-09-11 09:59:33 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-03-13 09:57:26 +00:00
|
|
|
|
public void BringOffline(CodexNodeGroup group, bool waitTillStopped)
|
2023-09-11 09:59:33 +00:00
|
|
|
|
{
|
2023-09-13 10:09:44 +00:00
|
|
|
|
Log($"Stopping {group.Describe()}...");
|
2023-09-20 10:55:09 +00:00
|
|
|
|
StopCrashWatcher(group);
|
2023-09-13 10:09:44 +00:00
|
|
|
|
var workflow = pluginTools.CreateWorkflow();
|
|
|
|
|
foreach (var c in group.Containers)
|
|
|
|
|
{
|
2024-03-13 09:57:26 +00:00
|
|
|
|
workflow.Stop(c, waitTillStopped);
|
2023-09-13 10:09:44 +00:00
|
|
|
|
}
|
|
|
|
|
Log("Stopped.");
|
2023-09-11 09:59:33 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-09-13 14:06:05 +00:00
|
|
|
|
public string GetCodexId()
|
|
|
|
|
{
|
|
|
|
|
if (versionResponse != null) return versionResponse.version;
|
|
|
|
|
return recipe.Image;
|
|
|
|
|
}
|
|
|
|
|
|
2023-11-13 08:37:45 +00:00
|
|
|
|
public string GetCodexRevision()
|
|
|
|
|
{
|
|
|
|
|
if (versionResponse != null) return versionResponse.revision;
|
|
|
|
|
return "unknown";
|
|
|
|
|
}
|
|
|
|
|
|
2023-09-19 11:58:45 +00:00
|
|
|
|
private StartupConfig CreateStartupConfig(CodexSetup codexSetup)
|
2023-09-11 14:57:57 +00:00
|
|
|
|
{
|
|
|
|
|
var startupConfig = new StartupConfig();
|
|
|
|
|
startupConfig.NameOverride = codexSetup.NameOverride;
|
|
|
|
|
startupConfig.Add(codexSetup);
|
|
|
|
|
return startupConfig;
|
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-25 06:47:19 +00:00
|
|
|
|
private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
|
2023-09-11 14:57:57 +00:00
|
|
|
|
{
|
|
|
|
|
var result = new List<RunningContainers>();
|
|
|
|
|
for (var i = 0; i < numberOfNodes; i++)
|
|
|
|
|
{
|
2023-09-12 08:31:55 +00:00
|
|
|
|
var workflow = pluginTools.CreateWorkflow();
|
2023-09-11 14:57:57 +00:00
|
|
|
|
result.Add(workflow.Start(1, location, recipe, startupConfig));
|
|
|
|
|
}
|
|
|
|
|
return result.ToArray();
|
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-11-06 13:33:47 +00:00
|
|
|
|
private PodInfo GetPodInfo(RunningContainers rc)
|
|
|
|
|
{
|
|
|
|
|
var workflow = pluginTools.CreateWorkflow();
|
|
|
|
|
return workflow.GetPodInfo(rc);
|
|
|
|
|
}
|
|
|
|
|
|
2023-09-19 11:58:45 +00:00
|
|
|
|
private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
|
2023-09-11 14:57:57 +00:00
|
|
|
|
{
|
2023-09-13 10:09:44 +00:00
|
|
|
|
var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory);
|
2023-09-11 14:57:57 +00:00
|
|
|
|
|
|
|
|
|
try
|
|
|
|
|
{
|
2023-09-13 08:23:05 +00:00
|
|
|
|
Stopwatch.Measure(pluginTools.GetLog(), "EnsureOnline", group.EnsureOnline);
|
2023-09-11 14:57:57 +00:00
|
|
|
|
}
|
|
|
|
|
catch
|
|
|
|
|
{
|
2023-09-19 11:58:45 +00:00
|
|
|
|
CodexNodesNotOnline(coreInterface, runningContainers);
|
2023-09-11 14:57:57 +00:00
|
|
|
|
throw;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return group;
|
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-19 11:58:45 +00:00
|
|
|
|
private void CodexNodesNotOnline(CoreInterface coreInterface, RunningContainers[] runningContainers)
|
2023-09-11 14:57:57 +00:00
|
|
|
|
{
|
2023-09-13 07:12:18 +00:00
|
|
|
|
Log("Codex nodes failed to start");
|
2023-09-19 11:58:45 +00:00
|
|
|
|
foreach (var container in runningContainers.Containers()) coreInterface.DownloadLog(container);
|
2023-09-11 14:57:57 +00:00
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-12 13:18:36 +00:00
|
|
|
|
private void LogSeparator()
|
|
|
|
|
{
|
|
|
|
|
Log("----------------------------------------------------------------------------");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void Log(string message)
|
|
|
|
|
{
|
2023-09-13 08:23:05 +00:00
|
|
|
|
pluginTools.GetLog().Log(message);
|
2023-09-12 13:18:36 +00:00
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-20 10:55:09 +00:00
|
|
|
|
private void StopCrashWatcher(CodexNodeGroup group)
|
2023-09-13 10:09:44 +00:00
|
|
|
|
{
|
2023-09-20 10:55:09 +00:00
|
|
|
|
foreach (var node in group)
|
2023-09-13 10:09:44 +00:00
|
|
|
|
{
|
2023-09-20 10:55:09 +00:00
|
|
|
|
node.CrashWatcher.Stop();
|
2023-09-13 10:09:44 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
}
|
|
|
|
|
}
|