2024-07-26 06:39:27 +00:00
|
|
|
|
using CodexPlugin.Hooks;
|
|
|
|
|
using Core;
|
2024-08-21 08:45:17 +00:00
|
|
|
|
using GethPlugin;
|
2023-09-11 14:57:57 +00:00
|
|
|
|
using KubernetesWorkflow;
|
2023-11-12 09:07:23 +00:00
|
|
|
|
using KubernetesWorkflow.Types;
|
2023-09-11 09:59:33 +00:00
|
|
|
|
using Logging;
|
|
|
|
|
|
|
|
|
|
namespace CodexPlugin
|
|
|
|
|
{
|
2023-09-11 14:57:57 +00:00
|
|
|
|
public class CodexStarter
|
2023-09-11 09:59:33 +00:00
|
|
|
|
{
|
2023-09-12 08:31:55 +00:00
|
|
|
|
private readonly IPluginTools pluginTools;
|
2023-09-13 14:06:05 +00:00
|
|
|
|
private readonly CodexContainerRecipe recipe = new CodexContainerRecipe();
|
2024-03-26 13:42:47 +00:00
|
|
|
|
private readonly ApiChecker apiChecker;
|
2024-03-26 09:03:52 +00:00
|
|
|
|
private DebugInfoVersion? versionResponse;
|
2023-09-11 14:57:57 +00:00
|
|
|
|
|
2023-09-12 13:18:36 +00:00
|
|
|
|
public CodexStarter(IPluginTools pluginTools)
|
2023-09-11 14:57:57 +00:00
|
|
|
|
{
|
2023-09-12 13:18:36 +00:00
|
|
|
|
this.pluginTools = pluginTools;
|
2024-03-26 13:42:47 +00:00
|
|
|
|
|
|
|
|
|
apiChecker = new ApiChecker(pluginTools);
|
2023-09-11 14:57:57 +00:00
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2024-07-26 06:39:27 +00:00
|
|
|
|
public CodexHooksFactory HooksFactory { get; } = new CodexHooksFactory();
|
|
|
|
|
|
2024-04-13 14:09:17 +00:00
|
|
|
|
public RunningPod[] BringOnline(CodexSetup codexSetup)
|
2023-09-11 09:59:33 +00:00
|
|
|
|
{
|
2023-09-12 13:18:36 +00:00
|
|
|
|
LogSeparator();
|
|
|
|
|
Log($"Starting {codexSetup.Describe()}...");
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-19 11:58:45 +00:00
|
|
|
|
var startupConfig = CreateStartupConfig(codexSetup);
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-12 13:18:36 +00:00
|
|
|
|
var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2024-03-26 13:42:47 +00:00
|
|
|
|
apiChecker.CheckCompatibility(containers);
|
|
|
|
|
|
2023-11-06 13:33:47 +00:00
|
|
|
|
foreach (var rc in containers)
|
|
|
|
|
{
|
|
|
|
|
var podInfo = GetPodInfo(rc);
|
2024-06-12 08:48:52 +00:00
|
|
|
|
var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' PodLabel: '{c.RunningPod.StartResult.Deployment.PodLabel}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
|
2024-07-25 08:10:11 +00:00
|
|
|
|
Log($"Started node with image '{containers.First().Containers.First().Recipe.Image}'. ({podInfos})");
|
2024-08-21 08:45:17 +00:00
|
|
|
|
LogEthAddress(rc);
|
2023-11-06 13:33:47 +00:00
|
|
|
|
}
|
2023-09-12 13:18:36 +00:00
|
|
|
|
LogSeparator();
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-12 13:18:36 +00:00
|
|
|
|
return containers;
|
2023-09-11 14:57:57 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-04-13 14:09:17 +00:00
|
|
|
|
public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
|
2023-09-11 14:57:57 +00:00
|
|
|
|
{
|
2024-07-26 06:39:27 +00:00
|
|
|
|
var codexNodeFactory = new CodexNodeFactory(pluginTools, HooksFactory);
|
2023-09-11 14:57:57 +00:00
|
|
|
|
|
2023-09-19 11:58:45 +00:00
|
|
|
|
var group = CreateCodexGroup(coreInterface, containers, codexNodeFactory);
|
2023-09-11 14:57:57 +00:00
|
|
|
|
|
2023-09-12 13:18:36 +00:00
|
|
|
|
Log($"Codex version: {group.Version}");
|
2023-09-13 14:06:05 +00:00
|
|
|
|
versionResponse = group.Version;
|
2023-09-12 13:18:36 +00:00
|
|
|
|
|
|
|
|
|
return group;
|
2023-09-11 09:59:33 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-03-13 09:57:26 +00:00
|
|
|
|
public void BringOffline(CodexNodeGroup group, bool waitTillStopped)
|
2023-09-11 09:59:33 +00:00
|
|
|
|
{
|
2023-09-13 10:09:44 +00:00
|
|
|
|
Log($"Stopping {group.Describe()}...");
|
2023-09-20 10:55:09 +00:00
|
|
|
|
StopCrashWatcher(group);
|
2023-09-13 10:09:44 +00:00
|
|
|
|
var workflow = pluginTools.CreateWorkflow();
|
|
|
|
|
foreach (var c in group.Containers)
|
|
|
|
|
{
|
2024-03-13 09:57:26 +00:00
|
|
|
|
workflow.Stop(c, waitTillStopped);
|
2023-09-13 10:09:44 +00:00
|
|
|
|
}
|
|
|
|
|
Log("Stopped.");
|
2023-09-11 09:59:33 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-04-13 14:09:17 +00:00
|
|
|
|
public void Stop(RunningPod pod, bool waitTillStopped)
|
|
|
|
|
{
|
|
|
|
|
Log($"Stopping node...");
|
|
|
|
|
var workflow = pluginTools.CreateWorkflow();
|
|
|
|
|
workflow.Stop(pod, waitTillStopped);
|
|
|
|
|
Log("Stopped.");
|
|
|
|
|
}
|
|
|
|
|
|
2023-09-13 14:06:05 +00:00
|
|
|
|
public string GetCodexId()
|
|
|
|
|
{
|
2024-03-26 07:58:16 +00:00
|
|
|
|
if (versionResponse != null) return versionResponse.Version;
|
2023-09-13 14:06:05 +00:00
|
|
|
|
return recipe.Image;
|
|
|
|
|
}
|
|
|
|
|
|
2023-11-13 08:37:45 +00:00
|
|
|
|
public string GetCodexRevision()
|
|
|
|
|
{
|
2024-03-26 07:58:16 +00:00
|
|
|
|
if (versionResponse != null) return versionResponse.Revision;
|
2023-11-13 08:37:45 +00:00
|
|
|
|
return "unknown";
|
|
|
|
|
}
|
|
|
|
|
|
2023-09-19 11:58:45 +00:00
|
|
|
|
private StartupConfig CreateStartupConfig(CodexSetup codexSetup)
|
2023-09-11 14:57:57 +00:00
|
|
|
|
{
|
|
|
|
|
var startupConfig = new StartupConfig();
|
|
|
|
|
startupConfig.NameOverride = codexSetup.NameOverride;
|
|
|
|
|
startupConfig.Add(codexSetup);
|
|
|
|
|
return startupConfig;
|
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2024-04-13 14:09:17 +00:00
|
|
|
|
private RunningPod[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location)
|
2023-09-11 14:57:57 +00:00
|
|
|
|
{
|
2024-04-09 08:23:07 +00:00
|
|
|
|
var futureContainers = new List<FutureContainers>();
|
2023-09-11 14:57:57 +00:00
|
|
|
|
for (var i = 0; i < numberOfNodes; i++)
|
|
|
|
|
{
|
2023-09-12 08:31:55 +00:00
|
|
|
|
var workflow = pluginTools.CreateWorkflow();
|
2024-04-09 08:23:07 +00:00
|
|
|
|
futureContainers.Add(workflow.Start(1, location, recipe, startupConfig));
|
2023-09-11 14:57:57 +00:00
|
|
|
|
}
|
2024-04-09 07:30:45 +00:00
|
|
|
|
|
2024-04-09 08:23:07 +00:00
|
|
|
|
return futureContainers
|
|
|
|
|
.Select(f => f.WaitForOnline())
|
|
|
|
|
.ToArray();
|
2023-09-11 14:57:57 +00:00
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2024-04-13 14:09:17 +00:00
|
|
|
|
private PodInfo GetPodInfo(RunningPod rc)
|
2023-11-06 13:33:47 +00:00
|
|
|
|
{
|
|
|
|
|
var workflow = pluginTools.CreateWorkflow();
|
|
|
|
|
return workflow.GetPodInfo(rc);
|
|
|
|
|
}
|
|
|
|
|
|
2024-04-13 14:09:17 +00:00
|
|
|
|
private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningPod[] runningContainers, CodexNodeFactory codexNodeFactory)
|
2023-09-11 14:57:57 +00:00
|
|
|
|
{
|
2023-09-13 10:09:44 +00:00
|
|
|
|
var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory);
|
2023-09-11 14:57:57 +00:00
|
|
|
|
|
|
|
|
|
try
|
|
|
|
|
{
|
2023-09-13 08:23:05 +00:00
|
|
|
|
Stopwatch.Measure(pluginTools.GetLog(), "EnsureOnline", group.EnsureOnline);
|
2023-09-11 14:57:57 +00:00
|
|
|
|
}
|
|
|
|
|
catch
|
|
|
|
|
{
|
2023-09-19 11:58:45 +00:00
|
|
|
|
CodexNodesNotOnline(coreInterface, runningContainers);
|
2023-09-11 14:57:57 +00:00
|
|
|
|
throw;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return group;
|
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2024-04-13 14:09:17 +00:00
|
|
|
|
private void CodexNodesNotOnline(CoreInterface coreInterface, RunningPod[] runningContainers)
|
2023-09-11 14:57:57 +00:00
|
|
|
|
{
|
2023-09-13 07:12:18 +00:00
|
|
|
|
Log("Codex nodes failed to start");
|
2024-04-13 14:09:17 +00:00
|
|
|
|
foreach (var container in runningContainers.First().Containers) coreInterface.DownloadLog(container);
|
2023-09-11 14:57:57 +00:00
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-12 13:18:36 +00:00
|
|
|
|
private void LogSeparator()
|
|
|
|
|
{
|
|
|
|
|
Log("----------------------------------------------------------------------------");
|
|
|
|
|
}
|
|
|
|
|
|
2024-08-21 08:45:17 +00:00
|
|
|
|
private void LogEthAddress(RunningPod rc)
|
|
|
|
|
{
|
|
|
|
|
var account = rc.Containers.First().Recipe.Additionals.Get<EthAccount>();
|
|
|
|
|
if (account == null) return;
|
|
|
|
|
Log($"{rc.Name} = {account}");
|
|
|
|
|
}
|
|
|
|
|
|
2023-09-12 13:18:36 +00:00
|
|
|
|
private void Log(string message)
|
|
|
|
|
{
|
2023-09-13 08:23:05 +00:00
|
|
|
|
pluginTools.GetLog().Log(message);
|
2023-09-12 13:18:36 +00:00
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
|
2023-09-20 10:55:09 +00:00
|
|
|
|
private void StopCrashWatcher(CodexNodeGroup group)
|
2023-09-13 10:09:44 +00:00
|
|
|
|
{
|
2023-09-20 10:55:09 +00:00
|
|
|
|
foreach (var node in group)
|
2023-09-13 10:09:44 +00:00
|
|
|
|
{
|
2023-09-20 10:55:09 +00:00
|
|
|
|
node.CrashWatcher.Stop();
|
2023-09-13 10:09:44 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
2023-09-11 09:59:33 +00:00
|
|
|
|
}
|
|
|
|
|
}
|