Setting up Geth starters

CodexDistTestCore/K8sOperations.cs

@@ -343,47 +343,7 @@ namespace CodexDistTestCore
 
         private class CommandRunner
         {
-            private readonly Kubernetes client;
-            private readonly PodInfo pod;
-            private readonly string containerName;
-            private readonly string command;
-            private readonly string[] arguments;
-            private readonly List<string> lines = new List<string>();
             
-            public CommandRunner(Kubernetes client, PodInfo pod, string containerName, string command, string[] arguments)
-            {
-                this.client = client;
-                this.pod = pod;
-                this.containerName = containerName;
-                this.command = command;
-                this.arguments = arguments;
-            }
-
-            public void Run()
-            {
-                var input = new[] { command }.Concat(arguments).ToArray();
-
-                Utils.Wait(client.NamespacedPodExecAsync(
-                    pod.Name, K8sCluster.K8sNamespace, containerName, input, false, Callback, new CancellationToken()));
-            }
-
-            public string GetStdOut()
-            {
-                return string.Join(Environment.NewLine, lines);
-            }
-
-            private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr)
-            {
-                using var streamReader = new StreamReader(stdOut);
-                var line = streamReader.ReadLine();
-                while (line != null)
-                {
-                    lines.Add(line);
-                    line = streamReader.ReadLine();
-                }
-
-                return Task.CompletedTask;
-            }
         }
     }
 }

CodexDistTestCore/Marketplace/K8sGethSpecs.cs

@@ -6,15 +6,13 @@ namespace CodexDistTestCore.Marketplace
     public static class GethDockerImage
     {
         public const string Image = "thatbenbierens/geth-confenv:latest";
-        public const string AccountFilename = "account_string.txt";
-        public const string GenesisFilename = "genesis.json";
+
     }
 
     public class K8sGethBoostrapSpecs
     {
         public const string ContainerName = "dtest-gethb";
         private const string portName = "gethb";
-        private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9";
 
         public K8sGethBoostrapSpecs(int servicePort)
         {

DistTestCore/CodexStarter.cs

@@ -18,6 +18,8 @@ namespace DistTestCore
 
         public ICodexNodeGroup BringOnline(CodexSetup codexSetup)
         {
+            var something = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
+
             var containers = StartCodexContainers(codexSetup);
 
             var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers);

DistTestCore/DistTest.cs

@@ -1,4 +1,4 @@
-using DistTestCore.CodexLogsAndMetrics;
+using DistTestCore.Logs;
 using DistTestCore.Metrics;
 using NUnit.Framework;
 

DistTestCore/GethStarter.cs

@@ -0,0 +1,44 @@
+using DistTestCore.Marketplace;
+using KubernetesWorkflow;
+
+namespace DistTestCore
+{
+    public class GethStarter
+    {
+        private readonly TestLifecycle lifecycle;
+        private readonly WorkflowCreator workflowCreator;
+        private readonly GethBootstrapNodeStarter bootstrapNodeStarter;
+        private GethBootstrapNodeInfo? bootstrapNode;
+
+        public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+        {
+            this.lifecycle = lifecycle;
+            this.workflowCreator = workflowCreator;
+
+            bootstrapNodeStarter = new GethBootstrapNodeStarter(lifecycle, workflowCreator);
+        }
+
+        public object BringOnlineMarketplaceFor(CodexSetup codexSetup)
+        {
+            EnsureBootstrapNode();
+            StartCompanionNodes(codexSetup);
+            return null!;
+        }
+
+        private void EnsureBootstrapNode()
+        {
+            if (bootstrapNode != null) return;
+            bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode();
+        }
+
+        private void StartCompanionNodes(CodexSetup codexSetup)
+        {
+            throw new NotImplementedException();
+        }
+
+        private void Log(string msg)
+        {
+            lifecycle.Log.Log(msg);
+        }
+    }
+}

DistTestCore/Logs/CodexNodeLog.cs

@@ -1,7 +1,7 @@
 using Logging;
 using NUnit.Framework;
 
-namespace DistTestCore.CodexLogsAndMetrics
+namespace DistTestCore.Logs
 {
     public interface ICodexNodeLog
     {

DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs

@@ -1,6 +1,6 @@
 using NUnit.Framework;
 
-namespace DistTestCore.CodexLogsAndMetrics
+namespace DistTestCore.Logs
 {
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
     public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute

DistTestCore/Logs/LogDownloadHandler.cs

@@ -1,7 +1,7 @@
 using KubernetesWorkflow;
 using Logging;
 
-namespace DistTestCore.CodexLogsAndMetrics
+namespace DistTestCore.Logs
 {
     public class LogDownloadHandler : ILogHandler
     {

DistTestCore/Marketplace/GethBootstrapNodeInfo.cs

@@ -0,0 +1,18 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class GethBootstrapNodeInfo
+    {
+        public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64)
+        {
+            RunningContainers = runningContainers;
+            Account = account;
+            GenesisJsonBase64 = genesisJsonBase64;
+        }
+
+        public RunningContainers RunningContainers { get; }
+        public string Account { get; }
+        public string GenesisJsonBase64 { get; }
+    }
+}

DistTestCore/Marketplace/GethBootstrapNodeStarter.cs

@@ -0,0 +1,47 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class GethBootstrapNodeStarter
+    {
+        private const string bootstrapGenesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9";
+        private readonly TestLifecycle lifecycle;
+        private readonly WorkflowCreator workflowCreator;
+
+        public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+        {
+            this.lifecycle = lifecycle;
+            this.workflowCreator = workflowCreator;
+        }
+
+        public GethBootstrapNodeInfo StartGethBootstrapNode()
+        {
+            Log("Starting Geth bootstrap node...");
+            var startupConfig = CreateBootstrapStartupConfig();
+            
+            var workflow = workflowCreator.CreateWorkflow();
+            var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
+            if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
+
+            var extractor = new GethInfoExtractor(workflow, containers.Containers[0]);
+            var account = extractor.ExtractAccount();
+            var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64();
+
+            Log($"Geth bootstrap node started with account '{account}'");
+
+            return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64);
+        }
+
+        private StartupConfig CreateBootstrapStartupConfig()
+        {
+            var config = new StartupConfig();
+            config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64));
+            return config;
+        }
+
+        private void Log(string msg)
+        {
+            lifecycle.Log.Log(msg);
+        }
+    }
+}

DistTestCore/Marketplace/GethCompanionNodeInfo.cs

@@ -0,0 +1,16 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class GethCompanionNodeInfo
+    {
+        public GethCompanionNodeInfo(RunningContainer runningContainer, string account)
+        {
+            RunningContainer = runningContainer;
+            Account = account;
+        }
+
+        public RunningContainer RunningContainer { get; }
+        public string Account { get; }
+    }
+}

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -0,0 +1,50 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class GethCompanionNodeStarter
+    {
+        private readonly TestLifecycle lifecycle;
+        private readonly WorkflowCreator workflowCreator;
+
+        public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+        {
+            this.lifecycle = lifecycle;
+            this.workflowCreator = workflowCreator;
+        }
+
+        public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode)
+        {
+            Log($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes.");
+
+            var startupConfig = CreateCompanionNodeStartupConfig(bootstrapNode);
+
+            var workflow = workflowCreator.CreateWorkflow();
+            var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig);
+            if (containers.Containers.Length != codexSetup.NumberOfNodes) throw new InvalidOperationException("Expected a Geth companion node to be created for each Codex node. Test infra failure.");
+
+            Log("Initialized companion nodes.");
+
+            return containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray();
+        }
+
+        private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container)
+        {
+            var extractor = new GethInfoExtractor(workflow, container);
+            var account = extractor.ExtractAccount();
+            return new GethCompanionNodeInfo(container, account);
+        }
+
+        private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode)
+        {
+            var config = new StartupConfig();
+            config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64));
+            return config;
+        }
+
+        private void Log(string msg)
+        {
+            lifecycle.Log.Log(msg);
+        }
+    }
+}

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -0,0 +1,37 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class GethContainerRecipe : ContainerRecipeFactory
+    {
+        protected override string Image => "thatbenbierens/geth-confenv:latest";
+        public const string AccountFilename = "account_string.txt";
+        public const string GenesisFilename = "genesis.json";
+
+        protected override void Initialize(StartupConfig startupConfig)
+        {
+            var config = startupConfig.Get<GethStartupConfig>();
+
+            var args = CreateArgs(config);
+
+            AddEnvVar("GETH_ARGS", args);
+            AddEnvVar("GENESIS_JSON", config.GenesisJsonBase64);
+        }
+
+        private string CreateArgs(GethStartupConfig config)
+        {
+            if (config.IsBootstrapNode)
+            {
+                AddEnvVar("IS_BOOTSTRAP", "1");
+                var exposedPort = AddExposedPort();
+                return $"--http.port {exposedPort.Number}";
+            }
+
+            var port = AddInternalPort();
+            var discovery = AddInternalPort();
+            var authRpc = AddInternalPort();
+            var httpPort = AddInternalPort();
+            return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number}";
+        }
+    }
+}

DistTestCore/Marketplace/GethInfoExtractor.cs

@@ -0,0 +1,58 @@
+using KubernetesWorkflow;
+using System.Text;
+
+namespace DistTestCore.Marketplace
+{
+    public class GethInfoExtractor
+    {
+        private readonly StartupWorkflow workflow;
+        private readonly RunningContainer container;
+
+        public GethInfoExtractor(StartupWorkflow workflow, RunningContainer container)
+        {
+            this.workflow = workflow;
+            this.container = container;
+        }
+
+        public string ExtractAccount()
+        {
+            var account = Retry(FetchAccount);
+            
+            if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure.");
+
+            return account;
+        }
+
+        public string ExtractGenesisJsonBase64()
+        {
+            var genesisJson = Retry(FetchGenesisJson);
+
+            if (string.IsNullOrEmpty(genesisJson)) throw new InvalidOperationException("Unable to fetch genesis-json for geth node. Test infra failure.");
+
+            var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson));
+
+            return encoded;
+        }
+
+        private string Retry(Func<string> fetch)
+        {
+            var result = fetch();
+            if (string.IsNullOrEmpty(result))
+            {
+                Thread.Sleep(TimeSpan.FromSeconds(5));
+                result = fetch();
+            }
+            return result;
+        }
+
+        private string FetchGenesisJson()
+        {
+            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GenesisFilename);
+        }
+
+        private string FetchAccount()
+        {
+            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename);
+        }
+    }
+}

DistTestCore/Marketplace/GethStartupConfig.cs

@@ -0,0 +1,14 @@
+namespace DistTestCore.Marketplace
+{
+    public class GethStartupConfig
+    {
+        public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64)
+        {
+            IsBootstrapNode = isBootstrapNode;
+            GenesisJsonBase64 = genesisJsonBase64;
+        }
+
+        public bool IsBootstrapNode { get; }
+        public string GenesisJsonBase64 { get; }
+    }
+}

DistTestCore/OnlineCodexNode.cs

@@ -1,5 +1,5 @@
 using DistTestCore.Codex;
-using DistTestCore.CodexLogsAndMetrics;
+using DistTestCore.Logs;
 using DistTestCore.Metrics;
 using NUnit.Framework;
 

DistTestCore/TestLifecycle.cs

@@ -1,4 +1,4 @@
-using DistTestCore.CodexLogsAndMetrics;
+using DistTestCore.Logs;
 using KubernetesWorkflow;
 using Logging;
 
@@ -16,12 +16,14 @@ namespace DistTestCore
             FileManager = new FileManager(Log, configuration);
             CodexStarter = new CodexStarter(this, workflowCreator);
             PrometheusStarter = new PrometheusStarter(this, workflowCreator);
+            GethStarter = new GethStarter(this, workflowCreator);
         }
 
         public TestLog Log { get; }
         public FileManager FileManager { get; }
         public CodexStarter CodexStarter { get; }
         public PrometheusStarter PrometheusStarter { get; }
+        public GethStarter GethStarter { get; }
 
         public void DeleteAllResources()
         {

KubernetesWorkflow/CommandRunner.cs

@@ -0,0 +1,52 @@
+using k8s;
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public class CommandRunner
+    {
+        private readonly Kubernetes client;
+        private readonly string k8sNamespace;
+        private readonly RunningPod pod;
+        private readonly string containerName;
+        private readonly string command;
+        private readonly string[] arguments;
+        private readonly List<string> lines = new List<string>();
+
+        public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
+        {
+            this.client = client;
+            this.k8sNamespace = k8sNamespace;
+            this.pod = pod;
+            this.containerName = containerName;
+            this.command = command;
+            this.arguments = arguments;
+        }
+
+        public void Run()
+        {
+            var input = new[] { command }.Concat(arguments).ToArray();
+
+            Time.Wait(client.NamespacedPodExecAsync(
+                pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()));
+        }
+
+        public string GetStdOut()
+        {
+            return string.Join(Environment.NewLine, lines);
+        }
+
+        private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr)
+        {
+            using var streamReader = new StreamReader(stdOut);
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                lines.Add(line);
+                line = streamReader.ReadLine();
+            }
+
+            return Task.CompletedTask;
+        }
+    }
+}

KubernetesWorkflow/K8sController.cs

@@ -49,6 +49,13 @@ namespace KubernetesWorkflow
             logHandler.Log(stream);
         }
 
+        public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
+        {
+            var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
+            runner.Run();
+            return runner.GetStdOut();
+        }
+
         public void DeleteAllResources()
         {
             DeleteNamespace();

KubernetesWorkflow/StartupWorkflow.cs

@@ -42,6 +42,14 @@
             });
         }
 
+        public string ExecuteCommand(RunningContainer container, string command, params string[] args)
+        {
+            return K8s(controller =>
+            {
+                return controller.ExecuteCommand(container.Pod, container.Recipe.Name, command, args);
+            });
+        }
+
         public void DeleteAllResources()
         {
             K8s(controller =>