From ab8318e1022675c85af0d804613a64d72f8475ba Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 10 Apr 2023 14:48:16 +0200 Subject: [PATCH] Successful spin-up of geth companion node --- CodexDistTestCore/CodexNodeContainer.cs | 25 +++++++-- CodexDistTestCore/CodexNodeGroup.cs | 2 + CodexDistTestCore/K8sManager.cs | 8 +-- CodexDistTestCore/K8sOperations.cs | 17 ++++-- .../Marketplace/GethCompanionNodeContainer.cs | 53 +++++++++++++++++++ CodexDistTestCore/Marketplace/K8sGethSpecs.cs | 8 ++- .../Marketplace/MarketplaceController.cs | 26 +++++++-- Tests/BasicTests/SimpleTests.cs | 34 +++++------- 8 files changed, 136 insertions(+), 37 deletions(-) create mode 100644 CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs index 4a1f239..d9a87be 100644 --- a/CodexDistTestCore/CodexNodeContainer.cs +++ b/CodexDistTestCore/CodexNodeContainer.cs @@ -1,8 +1,10 @@ -namespace CodexDistTestCore +using CodexDistTestCore.Marketplace; + +namespace CodexDistTestCore { public class CodexNodeContainer { - public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort) + public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir, int metricsPort, GethCompanionNodeContainer? gethCompanionNodeContainer) { Name = name; ServicePort = servicePort; @@ -13,6 +15,7 @@ ListenPort = listenPort; DataDir = dataDir; MetricsPort = metricsPort; + GethCompanionNodeContainer = gethCompanionNodeContainer; } public string Name { get; } @@ -24,6 +27,8 @@ public int ListenPort { get; } public string DataDir { get; } public int MetricsPort { get; } + + public GethCompanionNodeContainer? GethCompanionNodeContainer { get; } } public class CodexGroupNumberSource @@ -71,7 +76,8 @@ discoveryPort: codexPortSource.GetNextNumber(), listenPort: codexPortSource.GetNextNumber(), dataDir: $"datadir{n}", - metricsPort: GetMetricsPort(offline) + metricsPort: GetMetricsPort(offline), + CreateGethNodeContainer(offline, n) ); } @@ -80,5 +86,18 @@ if (offline.MetricsEnabled) return codexPortSource.GetNextNumber(); return 0; } + + private GethCompanionNodeContainer? CreateGethNodeContainer(OfflineCodexNodes offline, int n) + { + if (offline.MarketplaceConfig == null) return null; + + return new GethCompanionNodeContainer( + name: $"geth-node{n}", + servicePort: groupContainerFactory.GetNextServicePort(), + servicePortName: groupContainerFactory.GetNextServicePortName(), + apiPort: codexPortSource.GetNextNumber(), + containerPortName: $"geth-{n}" + ); + } } } diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs index 975eccf..a3a82d3 100644 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -1,4 +1,5 @@ using CodexDistTestCore.Config; +using CodexDistTestCore.Marketplace; using k8s.Models; using System.Collections; @@ -45,6 +46,7 @@ namespace CodexDistTestCore public V1Deployment? Deployment { get; set; } public V1Service? Service { get; set; } public PodInfo? PodInfo { get; set; } + public GethInfo? GethInfo { get; set; } public CodexNodeContainer[] GetContainers() { diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 635a8fe..5e8716b 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -34,7 +34,7 @@ namespace CodexDistTestCore if (offline.MarketplaceConfig != null) { - BringOnlineMarketplace(); + online.GethInfo = BringOnlineMarketplace(); } K8s(k => k.BringOnline(online, offline)); @@ -87,7 +87,7 @@ namespace CodexDistTestCore return K8s(k => k.BringOnlinePrometheus(spec)); } - public PodInfo BringOnlineGethBootstrapNode() + public GethInfo BringOnlineGethBootstrapNode() { var spec = new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort()); @@ -104,9 +104,9 @@ namespace CodexDistTestCore metricsAggregator.BeginCollectingMetricsFor(DowncastNodes(group)); } - private void BringOnlineMarketplace() + private GethInfo BringOnlineMarketplace() { - marketplaceController.BringOnlineMarketplace(); + return marketplaceController.BringOnlineMarketplace(); } private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 86ab636..22e7998 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -2,10 +2,9 @@ using CodexDistTestCore.Marketplace; using CodexDistTestCore.Metrics; using k8s; -using k8s.KubeConfigModels; using k8s.Models; -using Nethereum.Merkle.Patricia; using NUnit.Framework; +using System.Numerics; namespace CodexDistTestCore { @@ -79,7 +78,7 @@ namespace CodexDistTestCore return new PrometheusInfo(spec.ServicePort, FetchNewPod()); } - public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) + public GethInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) { EnsureTestNamespace(); @@ -87,7 +86,7 @@ namespace CodexDistTestCore CreateGethBootstrapService(spec); WaitUntilGethBootstrapOnline(spec); - return FetchNewPod(); + return new GethInfo(spec, FetchNewPod()); } private void FetchPodInfo(CodexNodeGroup online) @@ -213,6 +212,11 @@ namespace CodexDistTestCore TargetPort = container.ContainerPortName, NodePort = container.ServicePort }); + + if (container.GethCompanionNodeContainer != null) + { + result.Add(container.GethCompanionNodeContainer.CreateServicePort()); + } } return result; } @@ -299,6 +303,11 @@ namespace CodexDistTestCore }, Env = dockerImage.CreateEnvironmentVariables(offline, container) }); + + if (container.GethCompanionNodeContainer != null) + { + result.Add(container.GethCompanionNodeContainer.CreateDeploymentContainer(online.GethInfo!)); + } } return result; diff --git a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs new file mode 100644 index 0000000..eeb84ca --- /dev/null +++ b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs @@ -0,0 +1,53 @@ +using k8s.Models; + +namespace CodexDistTestCore.Marketplace +{ + public class GethCompanionNodeContainer + { + public GethCompanionNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName) + { + Name = name; + ServicePort = servicePort; + ServicePortName = servicePortName; + ApiPort = apiPort; + ContainerPortName = containerPortName; + } + + public string Name { get; } + public int ServicePort { get; } + public string ServicePortName { get; } + public int ApiPort { get; } + public string ContainerPortName { get; } + + public V1Container CreateDeploymentContainer(GethInfo gethInfo) + { + return new V1Container + { + Name = Name, + Image = GethDockerImage.Image, + Ports = new List + { + new V1ContainerPort + { + ContainerPort = ApiPort, + Name = ContainerPortName + } + }, + // todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64 + //Env = dockerImage.CreateEnvironmentVariables(offline, container) + }; + } + + public V1ServicePort CreateServicePort() + { + return new V1ServicePort + { + Name = ServicePortName, + Protocol = "TCP", + Port = ApiPort, + TargetPort = ContainerPortName, + NodePort = ServicePort + }; + } + } +} diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index 0682b49..dd0204d 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -3,10 +3,14 @@ using k8s.Models; namespace CodexDistTestCore.Marketplace { + public static class GethDockerImage + { + public const string Image = "thatbenbierens/geth-confenv:latest"; + } + public class K8sGethBoostrapSpecs { public const string ContainerName = "dtest-gethb"; - private const string dockerImage = "thatbenbierens/geth-confenv:latest"; private const string portName = "gethb"; private const string genesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiAxMjM0NSwKICAgICAgImhvbWVzdGVhZEJsb2NrIjogMCwKICAgICAgImVpcDE1MEJsb2NrIjogMCwKICAgICAgImVpcDE1NUJsb2NrIjogMCwKICAgICAgImVpcDE1OEJsb2NrIjogMCwKICAgICAgImJ5emFudGl1bUJsb2NrIjogMCwKICAgICAgImNvbnN0YW50aW5vcGxlQmxvY2siOiAwLAogICAgICAicGV0ZXJzYnVyZ0Jsb2NrIjogMCwKICAgICAgImlzdGFuYnVsQmxvY2siOiAwLAogICAgICAibXVpckdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJiZXJsaW5CbG9jayI6IDAsCiAgICAgICJsb25kb25CbG9jayI6IDAsCiAgICAgICJhcnJvd0dsYWNpZXJCbG9jayI6IDAsCiAgICAgICJncmF5R2xhY2llckJsb2NrIjogMCwKICAgICAgImNsaXF1ZSI6IHsKICAgICAgICAicGVyaW9kIjogNSwKICAgICAgICAiZXBvY2giOiAzMDAwMAogICAgICB9CiAgICB9LAogICAgImRpZmZpY3VsdHkiOiAiMSIsCiAgICAiZ2FzTGltaXQiOiAiODAwMDAwMDAwIiwKICAgICJleHRyYWRhdGEiOiAiMHgwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwN2RmOWE4NzVhMTc0YjNiYzU2NWU2NDI0YTAwNTBlYmMxYjJkMWQ4MjAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiQUNDT1VOVF9IRVJFIjogeyAiYmFsYW5jZSI6ICI1MDAwMDAiIH0KICAgIH0KICB9"; @@ -52,7 +56,7 @@ namespace CodexDistTestCore.Marketplace new V1Container { Name = ContainerName, - Image = dockerImage, + Image = GethDockerImage.Image, Ports = new List { new V1ContainerPort diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index 38b5ad1..fd68ade 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -1,4 +1,5 @@ using NUnit.Framework; +using System.Text; namespace CodexDistTestCore.Marketplace { @@ -6,7 +7,7 @@ namespace CodexDistTestCore.Marketplace { private readonly TestLog log; private readonly K8sManager k8sManager; - private PodInfo? gethBootstrapNode; + private GethInfo? gethBootstrapNode; private string bootstrapAccount = string.Empty; private string bootstrapGenesisJson = string.Empty; @@ -16,14 +17,16 @@ namespace CodexDistTestCore.Marketplace this.k8sManager = k8sManager; } - public void BringOnlineMarketplace() + public GethInfo BringOnlineMarketplace() { - if (gethBootstrapNode != null) return; + if (gethBootstrapNode != null) return gethBootstrapNode; log.Log("Starting Geth bootstrap node..."); gethBootstrapNode = k8sManager.BringOnlineGethBootstrapNode(); ExtractAccountAndGenesisJson(); log.Log("Geth boothstrap node started."); + + return gethBootstrapNode; } private void ExtractAccountAndGenesisJson() @@ -33,11 +36,26 @@ namespace CodexDistTestCore.Marketplace Assert.That(bootstrapAccount, Is.Not.Empty, "Unable to fetch account for bootstrap geth node. Test infra failure."); Assert.That(bootstrapGenesisJson, Is.Not.Empty, "Unable to fetch genesis-json for bootstrap geth node. Test infra failure."); + + gethBootstrapNode!.GenesisJsonBase64 = Convert.ToBase64String(Encoding.ASCII.GetBytes(bootstrapGenesisJson)); } private string ExecuteCommand(string command, params string[] arguments) { - return k8sManager.ExecuteCommand(gethBootstrapNode!, K8sGethBoostrapSpecs.ContainerName, command, arguments); + return k8sManager.ExecuteCommand(gethBootstrapNode!.Pod, K8sGethBoostrapSpecs.ContainerName, command, arguments); } } + + public class GethInfo + { + public GethInfo(K8sGethBoostrapSpecs spec, PodInfo pod) + { + Spec = spec; + Pod = pod; + } + + public K8sGethBoostrapSpecs Spec { get; } + public PodInfo Pod { get; } + public string GenesisJsonBase64 { get; set; } = string.Empty; + } } diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 3785ac3..0812b78 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -6,14 +6,6 @@ namespace Tests.BasicTests [TestFixture] public class SimpleTests : DistTest { - [Test] - public void DoCommand() - { - var primary = SetupCodexNodes(1).BringOnline()[0]; - - k8sManager.ExampleOfCMD(primary); - } - [Test] public void TwoMetricsExample() { @@ -47,22 +39,24 @@ namespace Tests.BasicTests .EnableMarketplace(initialBalance: 20) .BringOnline()[0]; - var secondary = SetupCodexNodes(1) - .EnableMarketplace(initialBalance: 1000) - .BringOnline()[0]; + //var secondary = SetupCodexNodes(1) + // .EnableMarketplace(initialBalance: 1000) + // .BringOnline()[0]; - primary.ConnectToPeer(secondary); - primary.Marketplace.AdvertiseStorage(10.GB(), pricePerMBPerSecond: 0.01f, collateral: 20); + //primary.ConnectToPeer(secondary); + //primary.Marketplace.AdvertiseStorage(10.GB(), pricePerMBPerSecond: 0.01f, collateral: 20); - var testFile = GenerateTestFile(10.MB()); - var contentId = secondary.UploadFile(testFile); - secondary.Marketplace.AdvertiseContract(contentId, maxPricePerMBPerSecond: 0.02f, minRequiredCollateral: 10, minRequiredNumberOfDuplicates: 1); + //var testFile = GenerateTestFile(10.MB()); + //var contentId = secondary.UploadFile(testFile); + //secondary.Marketplace.AdvertiseContract(contentId, maxPricePerMBPerSecond: 0.02f, minRequiredCollateral: 10, minRequiredNumberOfDuplicates: 1); - primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); - var primaryBalance = primary.Marketplace.GetBalance(); + //primary.Marketplace.AssertThatBalance(Is.LessThan(20), "Collateral was not placed."); + //var primaryBalance = primary.Marketplace.GetBalance(); - secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); - primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); + //secondary.Marketplace.AssertThatBalance(Is.LessThan(1000), "Contractor was not charged for storage."); + //primary.Marketplace.AssertThatBalance(Is.GreaterThan(primaryBalance), "Storer was not paid for storage."); + + var aa = 0; } [Test]