diff --git a/Framework/KubernetesWorkflow/RunningContainers.cs b/Framework/KubernetesWorkflow/RunningContainers.cs index 4b36f95..cfcae4d 100644 --- a/Framework/KubernetesWorkflow/RunningContainers.cs +++ b/Framework/KubernetesWorkflow/RunningContainers.cs @@ -41,9 +41,6 @@ namespace KubernetesWorkflow public Address ClusterExternalAddress { get; } public Address ClusterInternalAddress { get; } - [JsonIgnore] - public CrashWatcher? CrashWatcher { get; set; } - [JsonIgnore] public Address Address { diff --git a/Framework/KubernetesWorkflow/StartupConfig.cs b/Framework/KubernetesWorkflow/StartupConfig.cs index 6aa7268..76a96b6 100644 --- a/Framework/KubernetesWorkflow/StartupConfig.cs +++ b/Framework/KubernetesWorkflow/StartupConfig.cs @@ -5,7 +5,6 @@ private readonly List configs = new List(); public string? NameOverride { get; set; } - public bool CreateCrashWatcher { get; set; } public void Add(object config) { diff --git a/Framework/KubernetesWorkflow/StartupWorkflow.cs b/Framework/KubernetesWorkflow/StartupWorkflow.cs index d8221d1..58f5273 100644 --- a/Framework/KubernetesWorkflow/StartupWorkflow.cs +++ b/Framework/KubernetesWorkflow/StartupWorkflow.cs @@ -6,6 +6,7 @@ namespace KubernetesWorkflow public interface IStartupWorkflow { RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig); + CrashWatcher CreateCrashWatcher(RunningContainer container); void Stop(RunningContainers runningContainers); void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null); string ExecuteCommand(RunningContainer container, string command, params string[] args); @@ -39,14 +40,17 @@ namespace KubernetesWorkflow var runningPod = controller.BringOnline(recipes, location); var containers = CreateContainers(runningPod, recipes, startupConfig); - if (startupConfig.CreateCrashWatcher) CreateCrashWatchers(controller, containers); - var rc = new RunningContainers(startupConfig, runningPod, containers); cluster.Configuration.Hooks.OnContainersStarted(rc); return rc; }); } + public CrashWatcher CreateCrashWatcher(RunningContainer container) + { + return K8s(c => c.CreateCrashWatcher(container)); + } + public void Stop(RunningContainers runningContainers) { K8s(controller => @@ -88,14 +92,6 @@ namespace KubernetesWorkflow }); } - private void CreateCrashWatchers(K8sController controller, RunningContainer[] runningContainers) - { - foreach (var container in runningContainers) - { - container.CrashWatcher = controller.CreateCrashWatcher(container); - } - } - private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig) { log.Debug(); diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index fa9c00a..4cb9390 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -8,16 +8,18 @@ namespace CodexPlugin private readonly IPluginTools tools; private bool hasContainerCrashed; - public CodexAccess(IPluginTools tools, RunningContainer container) + public CodexAccess(IPluginTools tools, RunningContainer container, CrashWatcher crashWatcher) { this.tools = tools; Container = container; + CrashWatcher = crashWatcher; hasContainerCrashed = false; - if (container.CrashWatcher != null) container.CrashWatcher.Start(this); + CrashWatcher.Start(this); } public RunningContainer Container { get; } + public CrashWatcher CrashWatcher { get; } public CodexDebugResponse GetDebugInfo() { diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index a47f2b2..f4ff802 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -18,6 +18,7 @@ namespace CodexPlugin void ConnectToPeer(ICodexNode node); CodexDebugVersionResponse Version { get; } IMarketplaceAccess Marketplace { get; } + CrashWatcher CrashWatcher { get; } void Stop(); } @@ -40,6 +41,7 @@ namespace CodexPlugin public RunningContainer Container { get { return CodexAccess.Container; } } public CodexAccess CodexAccess { get; } + public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; } public CodexNodeGroup Group { get; } public IMarketplaceAccess Marketplace { get; } public CodexDebugVersionResponse Version { get; private set; } diff --git a/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs b/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs index 29daf0a..4b9dfdd 100644 --- a/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs +++ b/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs @@ -1,11 +1,13 @@ using Core; using GethPlugin; +using KubernetesWorkflow; namespace CodexPlugin { public interface ICodexNodeFactory { CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group); + CrashWatcher CreateCrashWatcher(RunningContainer c); } public class CodexNodeFactory : ICodexNodeFactory @@ -36,5 +38,10 @@ namespace CodexPlugin if (mStart == null) return null; return mStart.EthAddress; } + + public CrashWatcher CreateCrashWatcher(RunningContainer c) + { + return tools.CreateWorkflow().CreateCrashWatcher(c); + } } } diff --git a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs b/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs index cff4f8c..01c5661 100644 --- a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs +++ b/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs @@ -76,7 +76,8 @@ namespace CodexPlugin private CodexNode CreateOnlineCodexNode(RunningContainer c, IPluginTools tools, ICodexNodeFactory factory) { - var access = new CodexAccess(tools, c); + var watcher = factory.CreateCrashWatcher(c); + var access = new CodexAccess(tools, c, watcher); return factory.CreateOnlineCodexNode(access, this); } } diff --git a/ProjectPlugins/CodexPlugin/CodexStarter.cs b/ProjectPlugins/CodexPlugin/CodexStarter.cs index 3c987da..855fb14 100644 --- a/ProjectPlugins/CodexPlugin/CodexStarter.cs +++ b/ProjectPlugins/CodexPlugin/CodexStarter.cs @@ -46,10 +46,10 @@ namespace CodexPlugin public void BringOffline(CodexNodeGroup group) { Log($"Stopping {group.Describe()}..."); + StopCrashWatcher(group); var workflow = pluginTools.CreateWorkflow(); foreach (var c in group.Containers) { - StopCrashWatcher(c); workflow.Stop(c); } Log("Stopped."); @@ -65,7 +65,6 @@ namespace CodexPlugin { var startupConfig = new StartupConfig(); startupConfig.NameOverride = codexSetup.NameOverride; - startupConfig.CreateCrashWatcher = true; startupConfig.Add(codexSetup); return startupConfig; } @@ -114,11 +113,11 @@ namespace CodexPlugin pluginTools.GetLog().Log(message); } - private void StopCrashWatcher(RunningContainers containers) + private void StopCrashWatcher(CodexNodeGroup group) { - foreach (var c in containers.Containers) + foreach (var node in group) { - c.CrashWatcher?.Stop(); + node.CrashWatcher.Stop(); } } } diff --git a/Tools/CodexNetDeployer/CodexNodeStarter.cs b/Tools/CodexNetDeployer/CodexNodeStarter.cs index f7cdd94..b89acf2 100644 --- a/Tools/CodexNetDeployer/CodexNodeStarter.cs +++ b/Tools/CodexNetDeployer/CodexNodeStarter.cs @@ -12,7 +12,7 @@ namespace CodexNetDeployer private readonly CoreInterface ci; private readonly IGethNode gethNode; private readonly ICodexContracts contracts; - private string bootstrapSpr = ""; + private ICodexNode? bootstrapNode = null; private int validatorsLeft; public CodexNodeStarter(Configuration config, CoreInterface ci, IGethNode gethNode, ICodexContracts contracts, int numberOfValidators) @@ -27,7 +27,7 @@ namespace CodexNetDeployer public CodexNodeStartResult? Start(int i) { var name = GetCodexContainerName(i); - Console.Write($" - {i} ({name}) = "); + Console.Write($" - {i} ({name}) \t= "); ICodexNode? codexNode = null; try @@ -40,6 +40,7 @@ namespace CodexNetDeployer s.EnableMarketplace(gethNode, contracts, 100.Eth(), config.InitialTestTokens.TestTokens(), validatorsLeft > 0); s.EnableMetrics(); + if (bootstrapNode != null) s.WithBootstrapNode(bootstrapNode); if (config.BlockTTL != Configuration.SecondsIn1Day) s.WithBlockTTL(TimeSpan.FromSeconds(config.BlockTTL)); if (config.BlockMI != Configuration.TenMinutes) s.WithBlockMaintenanceInterval(TimeSpan.FromSeconds(config.BlockMI)); if (config.BlockMN != 1000) s.WithBlockMaintenanceNumber(config.BlockMN); @@ -60,8 +61,8 @@ namespace CodexNetDeployer { Console.Write("Storage available\tOK" + Environment.NewLine); - if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; validatorsLeft--; + if (bootstrapNode == null) bootstrapNode = codexNode; return new CodexNodeStartResult(codexNode); } } diff --git a/Tools/CodexNetDeployer/Configuration.cs b/Tools/CodexNetDeployer/Configuration.cs index 9eb107f..3ff1020 100644 --- a/Tools/CodexNetDeployer/Configuration.cs +++ b/Tools/CodexNetDeployer/Configuration.cs @@ -57,7 +57,7 @@ namespace CodexNetDeployer "set this option to override the label value.")] public string TestsTypePodLabel { get; set; } = "continuous-tests"; - [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment.")] + [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment. Default is false.")] public bool CheckPeerConnection { get; set; } = false; public List Validate() diff --git a/Tools/CodexNetDeployer/Deployer.cs b/Tools/CodexNetDeployer/Deployer.cs index 94b878d..ee9ff9a 100644 --- a/Tools/CodexNetDeployer/Deployer.cs +++ b/Tools/CodexNetDeployer/Deployer.cs @@ -117,7 +117,7 @@ namespace CodexNetDeployer Log("Starting container crash check..."); foreach (var startResult in startResults) { - var watcher = startResult.CodexNode.Container.CrashWatcher; + var watcher = startResult.CodexNode.CrashWatcher; if (watcher == null) throw new Exception("Expected each CodexNode container to be created with a crash-watcher."); if (watcher.HasContainerCrashed()) crashes.Add(startResult.CodexNode.Container); } diff --git a/Tools/CodexNetDeployer/deploy-continuous-testnet.sh b/Tools/CodexNetDeployer/deploy-continuous-testnet.sh index e4ad70a..6a3bbe8 100644 --- a/Tools/CodexNetDeployer/deploy-continuous-testnet.sh +++ b/Tools/CodexNetDeployer/deploy-continuous-testnet.sh @@ -12,5 +12,5 @@ dotnet run \ --block-ttl=180 \ --block-mi=120 \ --block-mn=10000 \ - --metrics=true \ + --metrics=1 \ --check-connect=1