diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs index a3a82d3..1ad5781 100644 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -46,7 +46,7 @@ namespace CodexDistTestCore public V1Deployment? Deployment { get; set; } public V1Service? Service { get; set; } public PodInfo? PodInfo { get; set; } - public GethInfo? GethInfo { get; set; } + public GethCompanionGroup? GethCompanionGroup { get; set; } public CodexNodeContainer[] GetContainers() { diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 5346c40..9686bcc 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -34,7 +34,7 @@ namespace CodexDistTestCore if (offline.MarketplaceConfig != null) { - group.GethInfo = marketplaceController.BringOnlineMarketplace(offline); + group.GethCompanionGroup = marketplaceController.BringOnlineMarketplace(offline); } K8s(k => k.BringOnline(group, offline)); @@ -91,13 +91,21 @@ namespace CodexDistTestCore return K8s(k => k.BringOnlinePrometheus(spec)); } - public GethInfo BringOnlineGethBootstrapNode() + public K8sGethBoostrapSpecs CreateGethBootstrapNodeSpec() { - var spec = new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort()); + return new K8sGethBoostrapSpecs(codexGroupNumberSource.GetNextServicePort()); + } + public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) + { return K8s(k => k.BringOnlineGethBootstrapNode(spec)); } + public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group) + { + return K8s(k => k.BringOnlineGethCompanionGroup(info, group)); + } + public void DownloadAllMetrics() { metricsAggregator.DownloadAllMetrics(); @@ -110,15 +118,15 @@ namespace CodexDistTestCore private void ConnectMarketplace(CodexNodeGroup group) { - foreach (var node in DowncastNodes(group)) + for (var i = 0; i < group.Nodes.Length; i++) { - ConnectMarketplace(group, node); + ConnectMarketplace(group, group.Nodes[i], group.GethCompanionGroup!.Containers[i]); } } - private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node) + private void ConnectMarketplace(CodexNodeGroup group, OnlineCodexNode node, GethCompanionNodeContainer container) { - var access = new MarketplaceAccess(this, marketplaceController, log, group, node.Container.GethCompanionNodeContainer!); + var access = new MarketplaceAccess(this, marketplaceController, log, group, container); access.Initialize(); node.Marketplace = access; } diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 22e7998..c0c7234 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -4,7 +4,6 @@ using CodexDistTestCore.Metrics; using k8s; using k8s.Models; using NUnit.Framework; -using System.Numerics; namespace CodexDistTestCore { @@ -78,7 +77,7 @@ namespace CodexDistTestCore return new PrometheusInfo(spec.ServicePort, FetchNewPod()); } - public GethInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) + public PodInfo BringOnlineGethBootstrapNode(K8sGethBoostrapSpecs spec) { EnsureTestNamespace(); @@ -86,7 +85,17 @@ namespace CodexDistTestCore CreateGethBootstrapService(spec); WaitUntilGethBootstrapOnline(spec); - return new GethInfo(spec, FetchNewPod()); + return FetchNewPod(); + } + + public PodInfo BringOnlineGethCompanionGroup(GethBootstrapInfo info, GethCompanionGroup group) + { + EnsureTestNamespace(); + + CreateGethCompanionDeployment(info, group); + WaitUntilGethCompanionGroupOnline(info.Spec, group); + + return FetchNewPod(); } private void FetchPodInfo(CodexNodeGroup online) @@ -148,7 +157,12 @@ namespace CodexDistTestCore private void WaitUntilGethBootstrapOnline(K8sGethBoostrapSpecs spec) { - WaitUntilDeploymentOnline(spec.GetDeploymentName()); + WaitUntilDeploymentOnline(spec.GetBootstrapDeploymentName()); + } + + private void WaitUntilGethCompanionGroupOnline(K8sGethBoostrapSpecs spec, GethCompanionGroup group) + { + WaitUntilDeploymentOnline(spec.GetCompanionDeploymentName(group)); } private void WaitUntilDeploymentOnline(string deploymentName) @@ -212,11 +226,6 @@ namespace CodexDistTestCore TargetPort = container.ContainerPortName, NodePort = container.ServicePort }); - - if (container.GethCompanionNodeContainer != null) - { - result.Add(container.GethCompanionNodeContainer.CreateServicePort()); - } } return result; } @@ -303,11 +312,6 @@ namespace CodexDistTestCore }, Env = dockerImage.CreateEnvironmentVariables(offline, container) }); - - if (container.GethCompanionNodeContainer != null) - { - result.Add(container.GethCompanionNodeContainer.CreateDeploymentContainer(online.GethInfo!)); - } } return result; @@ -330,6 +334,11 @@ namespace CodexDistTestCore client.CreateNamespacedDeployment(spec.CreateGethBootstrapDeployment(), K8sNamespace); } + private void CreateGethCompanionDeployment(GethBootstrapInfo info, GethCompanionGroup group) + { + client.CreateNamespacedDeployment(info.Spec.CreateGethCompanionDeployment(group, info), K8sNamespace); + } + #endregion #region Namespace management diff --git a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs index d1ded2e..de5e59e 100644 --- a/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs +++ b/CodexDistTestCore/Marketplace/GethCompanionNodeContainer.cs @@ -1,7 +1,18 @@ -using k8s.Models; - -namespace CodexDistTestCore.Marketplace +namespace CodexDistTestCore.Marketplace { + public class GethCompanionGroup + { + public GethCompanionGroup(int number, GethCompanionNodeContainer[] containers) + { + Number = number; + Containers = containers; + } + + public int Number { get; } + public GethCompanionNodeContainer[] Containers { get; } + public PodInfo? Pod { get; set; } + } + public class GethCompanionNodeContainer { public GethCompanionNodeContainer(string name, int apiPort, int rpcPort, string containerPortName) @@ -16,6 +27,7 @@ namespace CodexDistTestCore.Marketplace public int ApiPort { get; } public int RpcPort { get; } public string ContainerPortName { get; } - + + public string Account { get; set; } = string.Empty; } } diff --git a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs index 5247ce6..6318c4d 100644 --- a/CodexDistTestCore/Marketplace/K8sGethSpecs.cs +++ b/CodexDistTestCore/Marketplace/K8sGethSpecs.cs @@ -1,6 +1,5 @@ using CodexDistTestCore.Config; using k8s.Models; -using System.Xml.Linq; namespace CodexDistTestCore.Marketplace { @@ -24,11 +23,16 @@ namespace CodexDistTestCore.Marketplace public int ServicePort { get; } - public string GetDeploymentName() + public string GetBootstrapDeploymentName() { return "test-gethb"; } + public string GetCompanionDeploymentName(GethCompanionGroup group) + { + return "test-geth" + group.Number; + } + public V1Deployment CreateGethBootstrapDeployment() { var deploymentSpec = new V1Deployment @@ -36,7 +40,7 @@ namespace CodexDistTestCore.Marketplace ApiVersion = "apps/v1", Metadata = new V1ObjectMeta { - Name = GetDeploymentName(), + Name = GetBootstrapDeploymentName(), NamespaceProperty = K8sCluster.K8sNamespace }, Spec = new V1DeploymentSpec @@ -44,13 +48,13 @@ namespace CodexDistTestCore.Marketplace Replicas = 1, Selector = new V1LabelSelector { - MatchLabels = CreateSelector() + MatchLabels = CreateBootstrapSelector() }, Template = new V1PodTemplateSpec { Metadata = new V1ObjectMeta { - Labels = CreateSelector() + Labels = CreateBootstrapSelector() }, Spec = new V1PodSpec { @@ -109,7 +113,7 @@ namespace CodexDistTestCore.Marketplace Spec = new V1ServiceSpec { Type = "NodePort", - Selector = CreateSelector(), + Selector = CreateBootstrapSelector(), Ports = new List { new V1ServicePort @@ -127,18 +131,52 @@ namespace CodexDistTestCore.Marketplace return serviceSpec; } - public V1Deployment CreateGethCompanionDeployment(GethInfo gethInfo) + public V1Deployment CreateGethCompanionDeployment(GethCompanionGroup group, GethBootstrapInfo info) + { + var deploymentSpec = new V1Deployment + { + ApiVersion = "apps/v1", + Metadata = new V1ObjectMeta + { + Name = GetCompanionDeploymentName(group), + NamespaceProperty = K8sCluster.K8sNamespace + }, + Spec = new V1DeploymentSpec + { + Replicas = 1, + Selector = new V1LabelSelector + { + MatchLabels = CreateCompanionSelector() + }, + Template = new V1PodTemplateSpec + { + Metadata = new V1ObjectMeta + { + Labels = CreateCompanionSelector() + }, + Spec = new V1PodSpec + { + Containers = group.Containers.Select(c => CreateContainer(c, info)).ToList() + } + } + } + }; + + return deploymentSpec; + } + + private static V1Container CreateContainer(GethCompanionNodeContainer container, GethBootstrapInfo info) { return new V1Container { - Name = Name, + Name = container.Name, Image = GethDockerImage.Image, Ports = new List { new V1ContainerPort { - ContainerPort = ApiPort, - Name = ContainerPortName + ContainerPort = container.ApiPort, + Name = container.ContainerPortName } }, // todo: use env vars to connect this node to the bootstrap node provided by gethInfo.podInfo & gethInfo.servicePort & gethInfo.genesisJsonBase64 @@ -147,20 +185,25 @@ namespace CodexDistTestCore.Marketplace new V1EnvVar { Name = "GETH_ARGS", - Value = $"--port {ApiPort} --discovery.port {ApiPort} --authrpc.port {RpcPort}" + Value = $"--port {container.ApiPort} --discovery.port {container.ApiPort} --authrpc.port {container.RpcPort}" }, new V1EnvVar { Name = "GENESIS_JSON", - Value = gethInfo.GenesisJsonBase64 + Value = info.GenesisJsonBase64 } } }; } - private Dictionary CreateSelector() + private Dictionary CreateBootstrapSelector() { return new Dictionary { { "test-gethb", "dtest-gethb" } }; } + + private Dictionary CreateCompanionSelector() + { + return new Dictionary { { "test-gethc", "dtest-gethc" } }; + } } } diff --git a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs index 248e4ac..7782634 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceAccess.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceAccess.cs @@ -17,38 +17,37 @@ namespace CodexDistTestCore.Marketplace private readonly MarketplaceController marketplaceController; private readonly TestLog log; private readonly CodexNodeGroup group; - private readonly GethCompanionNodeContainer gethCompanionNodeContainer; - private string account = string.Empty; + private readonly GethCompanionNodeContainer container; public MarketplaceAccess( K8sManager k8sManager, MarketplaceController marketplaceController, TestLog log, - CodexNodeGroup group, - GethCompanionNodeContainer gethCompanionNodeContainer) + CodexNodeGroup group, + GethCompanionNodeContainer container) { this.k8sManager = k8sManager; this.marketplaceController = marketplaceController; this.log = log; this.group = group; - this.gethCompanionNodeContainer = gethCompanionNodeContainer; + this.container = container; } public void Initialize() { EnsureAccount(); - marketplaceController.AddToBalance(account, group.Origin.MarketplaceConfig!.InitialBalance); + marketplaceController.AddToBalance(container.Account, group.Origin.MarketplaceConfig!.InitialBalance); - log.Log($"Initialized Geth companion node with account '{account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}"); + log.Log($"Initialized Geth companion node with account '{container.Account}' and initial balance {group.Origin.MarketplaceConfig!.InitialBalance}"); } - public void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates) + public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) { throw new NotImplementedException(); } - public void MakeStorageAvailable(ByteSize size, float pricePerMBPerSecond, float collateral) + public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) { throw new NotImplementedException(); } @@ -66,28 +65,28 @@ namespace CodexDistTestCore.Marketplace private void EnsureAccount() { FetchAccount(); - if (string.IsNullOrEmpty(account)) + if (string.IsNullOrEmpty(container.Account)) { Thread.Sleep(TimeSpan.FromSeconds(15)); FetchAccount(); } - Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure."); + Assert.That(container.Account, Is.Not.Empty, "Unable to fetch account for geth companion node. Test infra failure."); } private void FetchAccount() { - account = k8sManager.ExecuteCommand(group.PodInfo!, gethCompanionNodeContainer.Name, "cat", GethDockerImage.AccountFilename); + container.Account = k8sManager.ExecuteCommand(group.PodInfo!, container.Name, "cat", GethDockerImage.AccountFilename); } } public class MarketplaceUnavailable : IMarketplaceAccess { - public void AdvertiseContract(ContentId contentId, float maxPricePerMBPerSecond, float minRequiredCollateral, float minRequiredNumberOfDuplicates) + public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes) { Unavailable(); } - public void MakeStorageAvailable(ByteSize size, float pricePerMBPerSecond, float collateral) + public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral) { Unavailable(); } diff --git a/CodexDistTestCore/Marketplace/MarketplaceController.cs b/CodexDistTestCore/Marketplace/MarketplaceController.cs index deaf747..9edea24 100644 --- a/CodexDistTestCore/Marketplace/MarketplaceController.cs +++ b/CodexDistTestCore/Marketplace/MarketplaceController.cs @@ -7,9 +7,9 @@ namespace CodexDistTestCore.Marketplace { private readonly TestLog log; private readonly K8sManager k8sManager; - private GethInfo? gethBootstrapNode; - private string bootstrapAccount = string.Empty; - private string bootstrapGenesisJson = string.Empty; + private readonly NumberSource companionGroupNumberSource = new NumberSource(0); + private List companionGroups = new List(); + private GethBootstrapInfo? bootstrapInfo; public MarketplaceController(TestLog log, K8sManager k8sManager) { @@ -17,30 +17,47 @@ namespace CodexDistTestCore.Marketplace this.k8sManager = k8sManager; } - public GethInfo BringOnlineMarketplace(OfflineCodexNodes offline) + public GethCompanionGroup BringOnlineMarketplace(OfflineCodexNodes offline) { - if (gethBootstrapNode != null) return gethBootstrapNode; + if (bootstrapInfo == null) + { + BringOnlineBootstrapNode(); + } - log.Log("Starting Geth bootstrap node..."); - gethBootstrapNode = k8sManager.BringOnlineGethBootstrapNode(); - ExtractAccountAndGenesisJson(); - log.Log($"Geth boothstrap node started. Initializing companions for {offline.NumberOfNodes} Codex nodes."); + log.Log($"Initializing companions for {offline.NumberOfNodes} Codex nodes."); + var group = new GethCompanionGroup(companionGroupNumberSource.GetNextNumber(), CreateCompanionContainers(offline)); + group.Pod = k8sManager.BringOnlineGethCompanionGroup(bootstrapInfo!, group); + companionGroups.Add(group); - - - return gethBootstrapNode; + log.Log("Initialized companion nodes."); + return group; } + private void BringOnlineBootstrapNode() + { + log.Log("Starting Geth bootstrap node..."); + var spec = k8sManager.CreateGethBootstrapNodeSpec(); + var pod = k8sManager.BringOnlineGethBootstrapNode(spec); + var (account, genesisJson) = ExtractAccountAndGenesisJson(); + bootstrapInfo = new GethBootstrapInfo(spec, pod, account, genesisJson); + log.Log($"Geth boothstrap node started."); + } - private GethCompanionNodeContainer? CreateGethNodeContainer(OfflineCodexNodes offline, int n) + private GethCompanionNodeContainer[] CreateCompanionContainers(OfflineCodexNodes offline) + { + var numberSource = new NumberSource(8080); + var result = new List(); + for (var i = 0; i < offline.NumberOfNodes; i++) result.Add(CreateGethNodeContainer(numberSource, i)); + return result.ToArray(); + } + + private GethCompanionNodeContainer CreateGethNodeContainer(NumberSource numberSource, int n) { return new GethCompanionNodeContainer( name: $"geth-node{n}", - servicePort: numberSource.GetNextServicePort(), - servicePortName: numberSource.GetNextServicePortName(), - apiPort: codexPortSource.GetNextNumber(), - rpcPort: codexPortSource.GetNextNumber(), + apiPort: numberSource.GetNextNumber(), + rpcPort: numberSource.GetNextNumber(), containerPortName: $"geth-{n}" ); } @@ -53,47 +70,50 @@ namespace CodexDistTestCore.Marketplace throw new NotImplementedException(); } - private void ExtractAccountAndGenesisJson() + private (string, string) ExtractAccountAndGenesisJson() { - FetchAccountAndGenesisJson(); - if (string.IsNullOrEmpty(bootstrapAccount) || string.IsNullOrEmpty(bootstrapGenesisJson)) + var (account, genesisJson) = FetchAccountAndGenesisJson(); + if (string.IsNullOrEmpty(account) || string.IsNullOrEmpty(genesisJson)) { Thread.Sleep(TimeSpan.FromSeconds(15)); - FetchAccountAndGenesisJson(); + (account, genesisJson) = FetchAccountAndGenesisJson(); } - Assert.That(bootstrapAccount, Is.Not.Empty, "Unable to fetch account for geth bootstrap node. Test infra failure."); - Assert.That(bootstrapGenesisJson, Is.Not.Empty, "Unable to fetch genesis-json for geth bootstrap node. Test infra failure."); + Assert.That(account, Is.Not.Empty, "Unable to fetch account for geth bootstrap node. Test infra failure."); + Assert.That(genesisJson, Is.Not.Empty, "Unable to fetch genesis-json for geth bootstrap node. Test infra failure."); - gethBootstrapNode!.GenesisJsonBase64 = Convert.ToBase64String(Encoding.ASCII.GetBytes(bootstrapGenesisJson)); + var encoded = Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson)); - log.Log($"Initialized geth bootstrap node with account '{bootstrapAccount}'"); + log.Log($"Initialized geth bootstrap node with account '{account}'"); + return (account, encoded); } - private void FetchAccountAndGenesisJson() + private (string, string) FetchAccountAndGenesisJson() { - bootstrapAccount = ExecuteCommand("cat", GethDockerImage.AccountFilename); - bootstrapGenesisJson = ExecuteCommand("cat", GethDockerImage.GenesisFilename); + var bootstrapAccount = ExecuteCommand("cat", GethDockerImage.AccountFilename); + var bootstrapGenesisJson = ExecuteCommand("cat", GethDockerImage.GenesisFilename); + return (bootstrapAccount, bootstrapGenesisJson); } private string ExecuteCommand(string command, params string[] arguments) { - return k8sManager.ExecuteCommand(gethBootstrapNode!.BootstrapPod, K8sGethBoostrapSpecs.ContainerName, command, arguments); + return k8sManager.ExecuteCommand(bootstrapInfo!.Pod, K8sGethBoostrapSpecs.ContainerName, command, arguments); } } - public class GethInfo + public class GethBootstrapInfo { - public GethInfo(K8sGethBoostrapSpecs spec, PodInfo bootstrapPod, PodInfo companionPod) + public GethBootstrapInfo(K8sGethBoostrapSpecs spec, PodInfo pod, string account, string genesisJsonBase64) { Spec = spec; - BootstrapPod = bootstrapPod; - CompanionPod = companionPod; + Pod = pod; + Account = account; + GenesisJsonBase64 = genesisJsonBase64; } public K8sGethBoostrapSpecs Spec { get; } - public PodInfo BootstrapPod { get; } - public PodInfo CompanionPod { get; } - public string GenesisJsonBase64 { get; set; } = string.Empty; + public PodInfo Pod { get; } + public string Account { get; } + public string GenesisJsonBase64 { get; } } }