From 4fd00607dfd42526bee153456376198e4b4b438e Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Apr 2023 09:54:07 +0200 Subject: [PATCH] Setting up Geth starters --- CodexDistTestCore/K8sOperations.cs | 42 +------------- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 4 +- DistTestCore/CodexStarter.cs | 2 + DistTestCore/DistTest.cs | 2 +- DistTestCore/GethStarter.cs | 44 ++++++++++++++ .../CodexNodeLog.cs | 2 +- ...ownloadLogsAndMetricsOnFailureAttribute.cs | 2 +- .../LogDownloadHandler.cs | 2 +- .../Marketplace/GethBootstrapNodeInfo.cs | 18 ++++++ .../Marketplace/GethBootstrapNodeStarter.cs | 47 +++++++++++++++ .../Marketplace/GethCompanionNodeInfo.cs | 16 +++++ .../Marketplace/GethCompanionNodeStarter.cs | 50 ++++++++++++++++ .../Marketplace/GethContainerRecipe.cs | 37 ++++++++++++ DistTestCore/Marketplace/GethInfoExtractor.cs | 58 +++++++++++++++++++ DistTestCore/Marketplace/GethStartupConfig.cs | 14 +++++ DistTestCore/OnlineCodexNode.cs | 2 +- DistTestCore/TestLifecycle.cs | 4 +- KubernetesWorkflow/CommandRunner.cs | 52 +++++++++++++++++ KubernetesWorkflow/K8sController.cs | 7 +++ KubernetesWorkflow/StartupWorkflow.cs | 8 +++ 20 files changed, 363 insertions(+), 50 deletions(-) create mode 100644 DistTestCore/GethStarter.cs rename DistTestCore/{CodexLogsAndMetrics => Logs}/CodexNodeLog.cs (95%) rename DistTestCore/{CodexLogsAndMetrics => Logs}/DontDownloadLogsAndMetricsOnFailureAttribute.cs (90%) rename DistTestCore/{CodexLogsAndMetrics => Logs}/LogDownloadHandler.cs (95%) create mode 100644 DistTestCore/Marketplace/GethBootstrapNodeInfo.cs create mode 100644 DistTestCore/Marketplace/GethBootstrapNodeStarter.cs create mode 100644 DistTestCore/Marketplace/GethCompanionNodeInfo.cs create mode 100644 DistTestCore/Marketplace/GethCompanionNodeStarter.cs create mode 100644 DistTestCore/Marketplace/GethContainerRecipe.cs create mode 100644 DistTestCore/Marketplace/GethInfoExtractor.cs create mode 100644 DistTestCore/Marketplace/GethStartupConfig.cs create mode 100644 KubernetesWorkflow/CommandRunner.cs diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 73f04b0..0cc3ea1 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -343,47 +343,7 @@ namespace CodexDistTestCore private class CommandRunner { - private readonly Kubernetes client; - private readonly PodInfo pod; - private readonly string containerName; - private readonly string command; - private readonly string[] arguments; - private readonly List lines = new List(); - - public CommandRunner(Kubernetes client, PodInfo pod, string containerName, string command, string[] arguments) - { - this.client = client; - this.pod = pod; - this.containerName = containerName; - this.command = command; - this.arguments = arguments; - } - - public void Run() - { - var input = new[] { command }.Concat(arguments).ToArray(); - - Utils.Wait(client.NamespacedPodExecAsync( - pod.Name, K8sCluster.K8sNamespace, containerName, input, false, Callback, new CancellationToken())); - } - - public string GetStdOut() - { - return string.Join(Environment.NewLine, lines); - } - - private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr) - { - using var streamReader = new StreamReader(stdOut); - var line = streamReader.ReadLine(); - while (line != null) - { - lines.Add(line); - line = streamReader.ReadLine(); - } - - return Task.CompletedTask; - } + } } } diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index f99d520..bfaf3d4 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -6,15 +6,13 @@ namespace CodexDistTestCore.Marketplace public static class GethDockerImage { public const string Image = "thatbenbierens/geth-confenv:latest"; - public const string AccountFilename = "account_string.txt"; - public const string GenesisFilename = "genesis.json"; + } public class K8sGethBoostrapSpecs { public const string ContainerName = "dtest-gethb"; private const string portName = "gethb"; - private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9"; public K8sGethBoostrapSpecs(int servicePort) { diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 65c9a32..b85fba4 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -18,6 +18,8 @@ namespace DistTestCore public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { + var something = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup); + var containers = StartCodexContainers(codexSetup); var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers); diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index a088e7c..e3631f4 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -1,4 +1,4 @@ -using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Logs; using DistTestCore.Metrics; using NUnit.Framework; diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs new file mode 100644 index 0000000..bf26523 --- /dev/null +++ b/DistTestCore/GethStarter.cs @@ -0,0 +1,44 @@ +using DistTestCore.Marketplace; +using KubernetesWorkflow; + +namespace DistTestCore +{ + public class GethStarter + { + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + private readonly GethBootstrapNodeStarter bootstrapNodeStarter; + private GethBootstrapNodeInfo? bootstrapNode; + + public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + + bootstrapNodeStarter = new GethBootstrapNodeStarter(lifecycle, workflowCreator); + } + + public object BringOnlineMarketplaceFor(CodexSetup codexSetup) + { + EnsureBootstrapNode(); + StartCompanionNodes(codexSetup); + return null!; + } + + private void EnsureBootstrapNode() + { + if (bootstrapNode != null) return; + bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode(); + } + + private void StartCompanionNodes(CodexSetup codexSetup) + { + throw new NotImplementedException(); + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + } +} diff --git a/DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs b/DistTestCore/Logs/CodexNodeLog.cs similarity index 95% rename from DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs rename to DistTestCore/Logs/CodexNodeLog.cs index a4a9cb0..ac92678 100644 --- a/DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs +++ b/DistTestCore/Logs/CodexNodeLog.cs @@ -1,7 +1,7 @@ using Logging; using NUnit.Framework; -namespace DistTestCore.CodexLogsAndMetrics +namespace DistTestCore.Logs { public interface ICodexNodeLog { diff --git a/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs b/DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs similarity index 90% rename from DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs rename to DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs index 0cf1cbe..b95d875 100644 --- a/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs +++ b/DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs @@ -1,6 +1,6 @@ using NUnit.Framework; -namespace DistTestCore.CodexLogsAndMetrics +namespace DistTestCore.Logs { [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute diff --git a/DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs b/DistTestCore/Logs/LogDownloadHandler.cs similarity index 95% rename from DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs rename to DistTestCore/Logs/LogDownloadHandler.cs index 59c14d8..b6136c3 100644 --- a/DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs +++ b/DistTestCore/Logs/LogDownloadHandler.cs @@ -1,7 +1,7 @@ using KubernetesWorkflow; using Logging; -namespace DistTestCore.CodexLogsAndMetrics +namespace DistTestCore.Logs { public class LogDownloadHandler : ILogHandler { diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs new file mode 100644 index 0000000..8f897ef --- /dev/null +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -0,0 +1,18 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethBootstrapNodeInfo + { + public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64) + { + RunningContainers = runningContainers; + Account = account; + GenesisJsonBase64 = genesisJsonBase64; + } + + public RunningContainers RunningContainers { get; } + public string Account { get; } + public string GenesisJsonBase64 { get; } + } +} diff --git a/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs new file mode 100644 index 0000000..7c114c4 --- /dev/null +++ b/DistTestCore/Marketplace/GethBootstrapNodeStarter.cs @@ -0,0 +1,47 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethBootstrapNodeStarter + { + private const string bootstrapGenesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9"; + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + + public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + } + + public GethBootstrapNodeInfo StartGethBootstrapNode() + { + Log("Starting Geth bootstrap node..."); + var startupConfig = CreateBootstrapStartupConfig(); + + var workflow = workflowCreator.CreateWorkflow(); + var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig); + if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure."); + + var extractor = new GethInfoExtractor(workflow, containers.Containers[0]); + var account = extractor.ExtractAccount(); + var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64(); + + Log($"Geth bootstrap node started with account '{account}'"); + + return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64); + } + + private StartupConfig CreateBootstrapStartupConfig() + { + var config = new StartupConfig(); + config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64)); + return config; + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + } +} diff --git a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs new file mode 100644 index 0000000..9b7bd23 --- /dev/null +++ b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs @@ -0,0 +1,16 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethCompanionNodeInfo + { + public GethCompanionNodeInfo(RunningContainer runningContainer, string account) + { + RunningContainer = runningContainer; + Account = account; + } + + public RunningContainer RunningContainer { get; } + public string Account { get; } + } +} diff --git a/DistTestCore/Marketplace/GethCompanionNodeStarter.cs b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs new file mode 100644 index 0000000..b8b178b --- /dev/null +++ b/DistTestCore/Marketplace/GethCompanionNodeStarter.cs @@ -0,0 +1,50 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethCompanionNodeStarter + { + private readonly TestLifecycle lifecycle; + private readonly WorkflowCreator workflowCreator; + + public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator) + { + this.lifecycle = lifecycle; + this.workflowCreator = workflowCreator; + } + + public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode) + { + Log($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes."); + + var startupConfig = CreateCompanionNodeStartupConfig(bootstrapNode); + + var workflow = workflowCreator.CreateWorkflow(); + var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig); + if (containers.Containers.Length != codexSetup.NumberOfNodes) throw new InvalidOperationException("Expected a Geth companion node to be created for each Codex node. Test infra failure."); + + Log("Initialized companion nodes."); + + return containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray(); + } + + private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container) + { + var extractor = new GethInfoExtractor(workflow, container); + var account = extractor.ExtractAccount(); + return new GethCompanionNodeInfo(container, account); + } + + private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode) + { + var config = new StartupConfig(); + config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64)); + return config; + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + } +} diff --git a/DistTestCore/Marketplace/GethContainerRecipe.cs b/DistTestCore/Marketplace/GethContainerRecipe.cs new file mode 100644 index 0000000..3df72f6 --- /dev/null +++ b/DistTestCore/Marketplace/GethContainerRecipe.cs @@ -0,0 +1,37 @@ +using KubernetesWorkflow; + +namespace DistTestCore.Marketplace +{ + public class GethContainerRecipe : ContainerRecipeFactory + { + protected override string Image => "thatbenbierens/geth-confenv:latest"; + public const string AccountFilename = "account_string.txt"; + public const string GenesisFilename = "genesis.json"; + + protected override void Initialize(StartupConfig startupConfig) + { + var config = startupConfig.Get(); + + var args = CreateArgs(config); + + AddEnvVar("GETH_ARGS", args); + AddEnvVar("GENESIS_JSON", config.GenesisJsonBase64); + } + + private string CreateArgs(GethStartupConfig config) + { + if (config.IsBootstrapNode) + { + AddEnvVar("IS_BOOTSTRAP", "1"); + var exposedPort = AddExposedPort(); + return $"--http.port {exposedPort.Number}"; + } + + var port = AddInternalPort(); + var discovery = AddInternalPort(); + var authRpc = AddInternalPort(); + var httpPort = AddInternalPort(); + return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number}"; + } + } +} diff --git a/DistTestCore/Marketplace/GethInfoExtractor.cs b/DistTestCore/Marketplace/GethInfoExtractor.cs new file mode 100644 index 0000000..f27e396 --- /dev/null +++ b/DistTestCore/Marketplace/GethInfoExtractor.cs @@ -0,0 +1,58 @@ +using KubernetesWorkflow; +using System.Text; + +namespace DistTestCore.Marketplace +{ + public class GethInfoExtractor + { + private readonly StartupWorkflow workflow; + private readonly RunningContainer container; + + public GethInfoExtractor(StartupWorkflow workflow, RunningContainer container) + { + this.workflow = workflow; + this.container = container; + } + + public string ExtractAccount() + { + var account = Retry(FetchAccount); + + if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure."); + + return account; + } + + public string ExtractGenesisJsonBase64() + { + var genesisJson = Retry(FetchGenesisJson); + + if (string.IsNullOrEmpty(genesisJson)) throw new InvalidOperationException("Unable to fetch genesis-json for geth node. Test infra failure."); + + var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); + + return encoded; + } + + private string Retry(Func fetch) + { + var result = fetch(); + if (string.IsNullOrEmpty(result)) + { + Thread.Sleep(TimeSpan.FromSeconds(5)); + result = fetch(); + } + return result; + } + + private string FetchGenesisJson() + { + return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GenesisFilename); + } + + private string FetchAccount() + { + return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename); + } + } +} diff --git a/DistTestCore/Marketplace/GethStartupConfig.cs b/DistTestCore/Marketplace/GethStartupConfig.cs new file mode 100644 index 0000000..60164bb --- /dev/null +++ b/DistTestCore/Marketplace/GethStartupConfig.cs @@ -0,0 +1,14 @@ +namespace DistTestCore.Marketplace +{ + public class GethStartupConfig + { + public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64) + { + IsBootstrapNode = isBootstrapNode; + GenesisJsonBase64 = genesisJsonBase64; + } + + public bool IsBootstrapNode { get; } + public string GenesisJsonBase64 { get; } + } +} diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 0efa51a..76ab592 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -1,5 +1,5 @@ using DistTestCore.Codex; -using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Logs; using DistTestCore.Metrics; using NUnit.Framework; diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index a5127a7..63d3c7a 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,4 +1,4 @@ -using DistTestCore.CodexLogsAndMetrics; +using DistTestCore.Logs; using KubernetesWorkflow; using Logging; @@ -16,12 +16,14 @@ namespace DistTestCore FileManager = new FileManager(Log, configuration); CodexStarter = new CodexStarter(this, workflowCreator); PrometheusStarter = new PrometheusStarter(this, workflowCreator); + GethStarter = new GethStarter(this, workflowCreator); } public TestLog Log { get; } public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } public PrometheusStarter PrometheusStarter { get; } + public GethStarter GethStarter { get; } public void DeleteAllResources() { diff --git a/KubernetesWorkflow/CommandRunner.cs b/KubernetesWorkflow/CommandRunner.cs new file mode 100644 index 0000000..a045500 --- /dev/null +++ b/KubernetesWorkflow/CommandRunner.cs @@ -0,0 +1,52 @@ +using k8s; +using Utils; + +namespace KubernetesWorkflow +{ + public class CommandRunner + { + private readonly Kubernetes client; + private readonly string k8sNamespace; + private readonly RunningPod pod; + private readonly string containerName; + private readonly string command; + private readonly string[] arguments; + private readonly List lines = new List(); + + public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments) + { + this.client = client; + this.k8sNamespace = k8sNamespace; + this.pod = pod; + this.containerName = containerName; + this.command = command; + this.arguments = arguments; + } + + public void Run() + { + var input = new[] { command }.Concat(arguments).ToArray(); + + Time.Wait(client.NamespacedPodExecAsync( + pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())); + } + + public string GetStdOut() + { + return string.Join(Environment.NewLine, lines); + } + + private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr) + { + using var streamReader = new StreamReader(stdOut); + var line = streamReader.ReadLine(); + while (line != null) + { + lines.Add(line); + line = streamReader.ReadLine(); + } + + return Task.CompletedTask; + } + } +} diff --git a/KubernetesWorkflow/K8sController.cs b/KubernetesWorkflow/K8sController.cs index 394f86e..0804e8f 100644 --- a/KubernetesWorkflow/K8sController.cs +++ b/KubernetesWorkflow/K8sController.cs @@ -49,6 +49,13 @@ namespace KubernetesWorkflow logHandler.Log(stream); } + public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args) + { + var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args); + runner.Run(); + return runner.GetStdOut(); + } + public void DeleteAllResources() { DeleteNamespace(); diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 9dff328..e78a5b2 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -42,6 +42,14 @@ }); } + public string ExecuteCommand(RunningContainer container, string command, params string[] args) + { + return K8s(controller => + { + return controller.ExecuteCommand(container.Pod, container.Recipe.Name, command, args); + }); + } + public void DeleteAllResources() { K8s(controller =>