From 8880ddd2bd8fc9acfcdc38c283c3b136e96e2729 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 17 Apr 2023 10:31:14 +0200 Subject: [PATCH] Attempting to set up geth bootstrap argument --- DistTestCore/Logs/LogDownloadHandler.cs | 19 +++----- .../Marketplace/GethBootstrapNodeInfo.cs | 6 ++- .../Marketplace/GethBootstrapNodeStarter.cs | 6 ++- .../Marketplace/GethCompanionNodeStarter.cs | 2 +- .../Marketplace/GethContainerRecipe.cs | 15 ++++-- DistTestCore/Marketplace/GethInfoExtractor.cs | 48 +++++++++++++++++-- DistTestCore/Marketplace/GethStartupConfig.cs | 4 +- KubernetesWorkflow/K8sController.cs | 2 +- KubernetesWorkflow/StartupWorkflow.cs | 20 +++++++- Tests/BasicTests/ExampleTests.cs | 2 +- Tests/BasicTests/OneClientTests.cs | 1 - 11 files changed, 96 insertions(+), 29 deletions(-) diff --git a/DistTestCore/Logs/LogDownloadHandler.cs b/DistTestCore/Logs/LogDownloadHandler.cs index a01ad9e..9cc1012 100644 --- a/DistTestCore/Logs/LogDownloadHandler.cs +++ b/DistTestCore/Logs/LogDownloadHandler.cs @@ -3,15 +3,16 @@ using Logging; namespace DistTestCore.Logs { - public class LogDownloadHandler : ILogHandler + public class LogDownloadHandler : LogHandler, ILogHandler { - private readonly string description; private readonly LogFile log; public LogDownloadHandler(string description, LogFile log) { - this.description = description; this.log = log; + + log.Write($"{description} -->> {log.FullFilename}"); + log.WriteRaw(description); } public CodexNodeLog CreateCodexNodeLog() @@ -19,17 +20,9 @@ namespace DistTestCore.Logs return new CodexNodeLog(log); } - public void Log(Stream stream) + protected override void ProcessLine(string line) { - log.Write($"{description} -->> {log.FullFilename}"); - log.WriteRaw(description); - var reader = new StreamReader(stream); - var line = reader.ReadLine(); - while (line != null) - { - log.WriteRaw(line); - line = reader.ReadLine(); - } + log.WriteRaw(line); } } } diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index 39b6715..46189c4 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -6,16 +6,20 @@ namespace DistTestCore.Marketplace { public class GethBootstrapNodeInfo { - public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64) + public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64, string pubKey, Port discoveryPort) { RunningContainers = runningContainers; Account = account; GenesisJsonBase64 = genesisJsonBase64; + PubKey = pubKey; + DiscoveryPort = discoveryPort; } public RunningContainers RunningContainers { get; } public string Account { get; } public string GenesisJsonBase64 { get; } + public string PubKey { get; } + public Port DiscoveryPort { get; } public NethereumInteraction StartInteraction(TestLog log) { diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs index 7c114c4..2868752 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -26,16 +26,18 @@ namespace DistTestCore.Marketplace var extractor = new GethInfoExtractor(workflow, containers.Containers[0]); var account = extractor.ExtractAccount(); var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64(); + var pubKey = extractor.ExtractPubKey(); + var discoveryPort = containers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag); Log($"Geth bootstrap node started with account '{account}'"); - return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64); + return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64, pubKey, discoveryPort); } private StartupConfig CreateBootstrapStartupConfig() { var config = new StartupConfig(); - config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64)); + config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64, null!)); return config; } diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs index b8b178b..c746bf1 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -38,7 +38,7 @@ namespace DistTestCore.Marketplace private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode) { var config = new StartupConfig(); - config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64)); + config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64, bootstrapNode)); return config; } diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs index 2dbfbde..fad225f 100644 --- a/DistTestCore/Marketplace/GethContainerRecipe.cs +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -6,6 +6,7 @@ namespace DistTestCore.Marketplace { public const string DockerImage = "thatbenbierens/geth-confenv:latest"; public const string HttpPortTag = "http_port"; + public const string DiscoveryPortTag = "disc_port"; public const string AccountFilename = "account_string.txt"; public const string GenesisFilename = "genesis.json"; @@ -23,18 +24,26 @@ namespace DistTestCore.Marketplace private string CreateArgs(GethStartupConfig config) { + var discovery = AddInternalPort(tag: DiscoveryPortTag); + if (config.IsBootstrapNode) { AddEnvVar("IS_BOOTSTRAP", "1"); var exposedPort = AddExposedPort(); - return $"--http.port {exposedPort.Number}"; + return $"--http.port {exposedPort.Number} --discovery.port {discovery.Number} --nodiscover"; } var port = AddInternalPort(); - var discovery = AddInternalPort(); var authRpc = AddInternalPort(); var httpPort = AddInternalPort(tag: HttpPortTag); - return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number}"; + + var bootPubKey = config.BootstrapNode.PubKey; + var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip; + var bootPort = config.BootstrapNode.DiscoveryPort.Number; + var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort}"; + // geth --bootnodes enode://pubkey1@ip1:port1 + + return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number} --nodiscover {bootstrapArg}"; } } } diff --git a/DistTestCore/Marketplace/GethInfoExtractor.cs b/DistTestCore/Marketplace/GethInfoExtractor.cs index f27e396..59725ec 100644 --- a/DistTestCore/Marketplace/GethInfoExtractor.cs +++ b/DistTestCore/Marketplace/GethInfoExtractor.cs @@ -17,7 +17,6 @@ namespace DistTestCore.Marketplace public string ExtractAccount() { var account = Retry(FetchAccount); - if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure."); return account; @@ -26,12 +25,17 @@ namespace DistTestCore.Marketplace public string ExtractGenesisJsonBase64() { var genesisJson = Retry(FetchGenesisJson); - if (string.IsNullOrEmpty(genesisJson)) throw new InvalidOperationException("Unable to fetch genesis-json for geth node. Test infra failure."); - var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); + return Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); + } - return encoded; + public string ExtractPubKey() + { + var pubKey = Retry(FetchPubKey); + if (string.IsNullOrEmpty(pubKey)) throw new InvalidOperationException("Unable to fetch enode from geth node. Test infra failure."); + + return pubKey; } private string Retry(Func fetch) @@ -54,5 +58,41 @@ namespace DistTestCore.Marketplace { return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename); } + + private string FetchPubKey() + { + var enodeFinder = new PubKeyFinder(); + workflow.DownloadContainerLog(container, enodeFinder); + return enodeFinder.GetPubKey(); + } + } + + public class PubKeyFinder : LogHandler, ILogHandler + { + private const string openTag = "self=\"enode://"; + private string pubKey = string.Empty; + + public string GetPubKey() + { + return pubKey; + } + + protected override void ProcessLine(string line) + { + if (line.Contains(openTag)) + { + ExtractPubKey(line); + } + } + + private void ExtractPubKey(string line) + { + var openIndex = line.IndexOf(openTag) + openTag.Length; + var closeIndex = line.IndexOf("@"); + + pubKey = line.Substring( + startIndex: openIndex, + length: closeIndex - openIndex); + } } } diff --git a/DistTestCore/Marketplace/GethStartupConfig.cs b/DistTestCore/Marketplace/GethStartupConfig.cs index 60164bb..baeb421 100644 --- a/DistTestCore/Marketplace/GethStartupConfig.cs +++ b/DistTestCore/Marketplace/GethStartupConfig.cs @@ -2,13 +2,15 @@ { public class GethStartupConfig { - public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64) + public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64, GethBootstrapNodeInfo bootstrapNode) { IsBootstrapNode = isBootstrapNode; GenesisJsonBase64 = genesisJsonBase64; + BootstrapNode = bootstrapNode; } public bool IsBootstrapNode { get; } public string GenesisJsonBase64 { get; } + public GethBootstrapNodeInfo BootstrapNode { get; } } } diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 0804e8f..6ce366a 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -45,7 +45,7 @@ namespace KubernetesWorkflow public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler) { - var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name); + using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name); logHandler.Log(stream); } diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index f575b15..d37699f 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -1,4 +1,6 @@ -namespace KubernetesWorkflow +using System.IO; + +namespace KubernetesWorkflow { public class StartupWorkflow { @@ -94,4 +96,20 @@ { void Log(Stream log); } + + public abstract class LogHandler : ILogHandler + { + public void Log(Stream log) + { + using var reader = new StreamReader(log); + var line = reader.ReadLine(); + while (line != null) + { + ProcessLine(line); + line = reader.ReadLine(); + } + } + + protected abstract void ProcessLine(string line); + } } diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index a3e3ae4..b70f3c4 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -49,7 +49,7 @@ namespace Tests.BasicTests [Test] public void MarketplaceExample() { - var group = SetupCodexNodes(4) + var group = SetupCodexNodes(2) .WithStorageQuota(10.GB()) .EnableMarketplace(initialBalance: 20) .BringOnline(); diff --git a/Tests/BasicTests/OneClientTests.cs b/Tests/BasicTests/OneClientTests.cs index 76ff79b..a2c0c57 100644 --- a/Tests/BasicTests/OneClientTests.cs +++ b/Tests/BasicTests/OneClientTests.cs @@ -15,7 +15,6 @@ namespace Tests.BasicTests } [Test] - [Ignore("Unstable.")] public void RestartTest() { var group = SetupCodexNodes(1).BringOnline();