From 124a4e373803a241c8454e5e6e279bb0a1b7afba Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 10 Apr 2023 10:09:41 +0200 Subject: [PATCH] setting up bootstrap geth node --- CodexDistTestCore/K8sManager.cs | 44 ++++-- CodexDistTestCore/K8sOperations.cs | 84 ++++++++---- CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 128 ++++++++++++++++++ .../Marketplace/MarketplaceController.cs | 38 ++++++ 4 files changed, 262 insertions(+), 32 deletions(-) create mode 100644 CodexDistTestCore/Marketplace/K8sGethSpecs.cs create mode 100644 CodexDistTestCore/Marketplace/MarketplaceController.cs diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 5ee9109..d10f8ea 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -1,4 +1,5 @@ -using CodexDistTestCore.Metrics; +using CodexDistTestCore.Marketplace; +using CodexDistTestCore.Metrics; namespace CodexDistTestCore { @@ -17,18 +18,25 @@ namespace CodexDistTestCore private readonly TestLog log; private readonly IFileManager fileManager; private readonly MetricsAggregator metricsAggregator; + private readonly MarketplaceController marketplaceController; public K8sManager(TestLog log, IFileManager fileManager) { this.log = log; this.fileManager = fileManager; metricsAggregator = new MetricsAggregator(log, this); + marketplaceController = new MarketplaceController(log, this); } public ICodexNodeGroup BringOnline(OfflineCodexNodes offline) { var online = CreateOnlineCodexNodes(offline); + if (offline.MarketplaceConfig != null) + { + BringOnlineMarketplace(); + } + K8s(k => k.BringOnline(online, offline)); log.Log($"{online.Describe()} online."); @@ -52,10 +60,9 @@ namespace CodexDistTestCore return online.Origin; } - public void ExampleOfCMD(IOnlineCodexNode node) + public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments) { - var n = (OnlineCodexNode)node; - K8s(k => k.ExampleOfCommandExecution(n)); + return K8s(k => k.ExecuteCommand(pod, containerName, command, arguments)); } public void DeleteAllResources() @@ -77,9 +84,12 @@ namespace CodexDistTestCore { var spec = new K8sPrometheusSpecs(codexGroupNumberSource.GetNextServicePort(), prometheusNumber, config); - PrometheusInfo? info = null; - K8s(k => info = k.BringOnlinePrometheus(spec)); - return info!; + return K8s(k => k.BringOnlinePrometheus(spec)); + } + + public PodInfo BringOnlineGethBootstrapNode() + { + return K8s(k => k.BringOnlineGethBootstrapNode()); } public void DownloadAllMetrics() @@ -89,9 +99,12 @@ namespace CodexDistTestCore private void BringOnlineMetrics(CodexNodeGroup group) { - var onlineNodes = group.Nodes.Cast().ToArray(); + metricsAggregator.BeginCollectingMetricsFor(DowncastNodes(group)); + } - metricsAggregator.BeginCollectingMetricsFor(onlineNodes); + private void BringOnlineMarketplace() + { + marketplaceController.BringOnlineMarketplace(); } private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) @@ -124,5 +137,18 @@ namespace CodexDistTestCore action(k8s); k8s.Close(); } + + private T K8s(Func action) + { + var k8s = new K8sOperations(knownPods); + var result = action(k8s); + k8s.Close(); + return result; + } + + private static OnlineCodexNode[] DowncastNodes(CodexNodeGroup group) + { + return group.Nodes.Cast().ToArray(); + } } } diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 2354e9d..576495c 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -3,6 +3,7 @@ using CodexDistTestCore.Metrics; using k8s; using k8s.KubeConfigModels; using k8s.Models; +using Nethereum.Merkle.Patricia; using NUnit.Framework; namespace CodexDistTestCore @@ -26,29 +27,6 @@ namespace CodexDistTestCore client.Dispose(); } - private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr) - { - using var streamReader = new StreamReader(stdOut); - var lines = new List(); - var line = streamReader.ReadLine(); - while (line != null) - { - lines.Add(line); - line = streamReader.ReadLine(); - } - - Assert.That(lines.Any(l => l.Contains("FOO76543"))); - - - return Task.CompletedTask; - } - - public void ExampleOfCommandExecution(OnlineCodexNode node) - { - Utils.Wait(client.NamespacedPodExecAsync( - node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name, new[] { "echo", "FOO76543" }, false, Callback, new CancellationToken())); - } - public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline) { EnsureTestNamespace(); @@ -82,6 +60,13 @@ namespace CodexDistTestCore logHandler.Log(stream); } + public string ExecuteCommand(PodInfo pod, string containerName, string command, params string[] arguments) + { + var runner = new CommandRunner(client, pod, containerName, command, arguments); + runner.Run(); + return runner.GetStdOut(); + } + public PrometheusInfo BringOnlinePrometheus(K8sPrometheusSpecs spec) { EnsureTestNamespace(); @@ -93,6 +78,14 @@ namespace CodexDistTestCore return new PrometheusInfo(spec.ServicePort, FetchNewPod()); } + public PodInfo BringOnlineGethBootstrapNode() + { + EnsureTestNamespace(); + + return FetchNewPod(); + + } + private void FetchPodInfo(CodexNodeGroup online) { online.PodInfo = FetchNewPod(); @@ -344,5 +337,50 @@ namespace CodexDistTestCore { return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); } + + private class CommandRunner + { + private readonly Kubernetes client; + private readonly PodInfo pod; + private readonly string containerName; + private readonly string command; + private readonly string[] arguments; + private readonly List lines = new List(); + + public CommandRunner(Kubernetes client, PodInfo pod, string containerName, string command, string[] arguments) + { + this.client = client; + this.pod = pod; + this.containerName = containerName; + this.command = command; + this.arguments = arguments; + } + + public void Run() + { + var input = new[] { command }.Concat(arguments).ToArray(); + + Utils.Wait(client.NamespacedPodExecAsync( + pod.Name, K8sCluster.K8sNamespace, containerName, input, false, Callback, new CancellationToken())); + } + + public string GetStdOut() + { + return string.Join(Environment.NewLine, lines); + } + + private Task Callback(Stream stdIn, Stream stdOut, Stream stdErr) + { + using var streamReader = new StreamReader(stdOut); + var line = streamReader.ReadLine(); + while (line != null) + { + lines.Add(line); + line = streamReader.ReadLine(); + } + + return Task.CompletedTask; + } + } } } diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs new file mode 100644 index 0000000..c2fd79e --- /dev/null +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -0,0 +1,128 @@ +using CodexDistTestCore.Config; +using k8s.Models; + +namespace CodexDistTestCore.Marketplace +{ + public class K8sGethBoostrapSpecs + { + public const string ContainerName = "dtest-gethb"; + private const string dockerImage = "thatbenbierens/prometheus-envconf:latest"; // todo - bake modified geth image and post it. + private const string portName = "gethb"; + private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiAxMjM0NSwKICAgICAgImhvbWVzdGVhZEJsb2NrIjogMCwKICAgICAgImVpcDE1MEJsb2NrIjogMCwKICAgICAgImVpcDE1NUJsb2NrIjogMCwKICAgICAgImVpcDE1OEJsb2NrIjogMCwKICAgICAgImJ5emFudGl1bUJsb2NrIjogMCwKICAgICAgImNvbnN0YW50aW5vcGxlQmxvY2siOiAwLAogICAgICAicGV0ZXJzYnVyZ0Jsb2NrIjogMCwKICAgICAgImlzdGFuYnVsQmxvY2siOiAwLAogICAgICAibXVpckdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJiZXJsaW5CbG9jayI6IDAsCiAgICAgICJsb25kb25CbG9jayI6IDAsCiAgICAgICJhcnJvd0dsYWNpZXJCbG9jayI6IDAsCiAgICAgICJncmF5R2xhY2llckJsb2NrIjogMCwKICAgICAgImNsaXF1ZSI6IHsKICAgICAgICAicGVyaW9kIjogNSwKICAgICAgICAiZXBvY2giOiAzMDAwMAogICAgICB9CiAgICB9LAogICAgImRpZmZpY3VsdHkiOiAiMSIsCiAgICAiZ2FzTGltaXQiOiAiODAwMDAwMDAwIiwKICAgICJleHRyYWRhdGEiOiAiMHgwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwN2RmOWE4NzVhMTc0YjNiYzU2NWU2NDI0YTAwNTBlYmMxYjJkMWQ4MjAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiQUNDT1VOVF9IRVJFIjogeyAiYmFsYW5jZSI6ICI1MDAwMDAiIH0KICAgIH0KICB9"; + + public K8sGethBoostrapSpecs(int servicePort) + { + ServicePort = servicePort; + } + + public int ServicePort { get; } + + public string GetDeploymentName() + { + return "test-gethb"; + } + + public V1Deployment CreateGethBootstrapDeployment() + { + var deploymentSpec = new V1Deployment + { + ApiVersion = "apps/v1", + Metadata = new V1ObjectMeta + { + Name = GetDeploymentName(), + NamespaceProperty = K8sCluster.K8sNamespace + }, + Spec = new V1DeploymentSpec + { + Replicas = 1, + Selector = new V1LabelSelector + { + MatchLabels = CreateSelector() + }, + Template = new V1PodTemplateSpec + { + Metadata = new V1ObjectMeta + { + Labels = CreateSelector() + }, + Spec = new V1PodSpec + { + Containers = new List + { + new V1Container + { + Name = ContainerName, + Image = dockerImage, + Ports = new List + { + new V1ContainerPort + { + ContainerPort = 9090, + Name = portName + } + }, + Env = new List + { + new V1EnvVar + { + Name = "GETH_ARGS", + Value = "--qwerty" + }, + new V1EnvVar + { + Name = "GENSIS_JSON", + Value = genesisJsonBase64 + }, + new V1EnvVar + { + Name = "IS_BOOTSTRAP", + Value = "1" + } + } + } + } + } + } + } + }; + + return deploymentSpec; + } + + public V1Service CreateGethBootstrapService() + { + var serviceSpec = new V1Service + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = "codex-gethb-service", + NamespaceProperty = K8sCluster.K8sNamespace + }, + Spec = new V1ServiceSpec + { + Type = "NodePort", + Selector = CreateSelector(), + Ports = new List + { + new V1ServicePort + { + Name = "gethb-service", + Protocol = "TCP", + Port = 9090, + TargetPort = portName, + NodePort = ServicePort + } + } + } + }; + + return serviceSpec; + } + + private Dictionary CreateSelector() + { + return new Dictionary { { "test-gethb", "dtest-gethb" } }; + } + } +} diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs new file mode 100644 index 0000000..d0f5947 --- /dev/null +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -0,0 +1,38 @@ +namespace CodexDistTestCore.Marketplace +{ + public class MarketplaceController + { + private readonly TestLog log; + private readonly K8sManager k8sManager; + private PodInfo? gethBootstrapNode; + private string bootstrapAccount = string.Empty; + private string bootstrapGenesisJson = string.Empty; + + public MarketplaceController(TestLog log, K8sManager k8sManager) + { + this.log = log; + this.k8sManager = k8sManager; + } + + public void BringOnlineMarketplace() + { + if (gethBootstrapNode != null) return; + + log.Log("Starting Geth bootstrap node..."); + gethBootstrapNode = k8sManager.BringOnlineGethBootstrapNode(); + ExtractAccountAndGenesisJson(); + log.Log("Geth boothstrap node started."); + } + + private void ExtractAccountAndGenesisJson() + { + bootstrapAccount = ExecuteCommand("cat", "account_string.txt"); + bootstrapGenesisJson = ExecuteCommand("cat", "genesis.json"); + } + + private string ExecuteCommand(string command, params string[] arguments) + { + return k8sManager.ExecuteCommand(gethBootstrapNode!, K8sGethBoostrapSpecs.ContainerName, command, arguments); + } + } +}