From b92c1b970d1bdb328962da3261e61f60827cf1af Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Jun 2023 09:35:18 +0200 Subject: [PATCH] Sets up support for internal and external container addresses --- DistTestCore/Codex/CodexAccess.cs | 20 ++++++------- DistTestCore/Codex/CodexContainerRecipe.cs | 4 +-- DistTestCore/CodexNodeGroup.cs | 2 +- DistTestCore/Configuration.cs | 20 +++++++++++++ DistTestCore/GethStarter.cs | 4 +-- DistTestCore/Http.cs | 13 ++++----- .../Marketplace/CodexContractsStarter.cs | 2 +- .../Marketplace/GethBootstrapNodeInfo.cs | 8 ++---- DistTestCore/Marketplace/MarketplaceAccess.cs | 10 +++---- .../Marketplace/MarketplaceAccessFactory.cs | 8 +++--- .../Marketplace/MarketplaceNetwork.cs | 7 ++--- DistTestCore/Metrics/MetricsAccessFactory.cs | 2 +- DistTestCore/Metrics/MetricsQuery.cs | 12 ++++---- DistTestCore/TestLifecycle.cs | 2 ++ KubernetesWorkflow/RunningContainers.cs | 18 +++++++++++- KubernetesWorkflow/StartupWorkflow.cs | 28 ++++++++++++++++++- Nethereum/NethereumInteraction.cs | 4 +-- Nethereum/NethereumInteractionCreator.cs | 2 +- Tests/BasicTests/ExampleTests.cs | 1 - 19 files changed, 111 insertions(+), 56 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 587f7f0..4d6f1da 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -1,17 +1,14 @@ using KubernetesWorkflow; -using Logging; namespace DistTestCore.Codex { public class CodexAccess { - private readonly BaseLog log; - private readonly ITimeSet timeSet; + private readonly TestLifecycle lifecycle; - public CodexAccess(BaseLog log, ITimeSet timeSet, RunningContainer runningContainer) + public CodexAccess(TestLifecycle lifecycle, RunningContainer runningContainer) { - this.log = log; - this.timeSet = timeSet; + this.lifecycle = lifecycle; Container = runningContainer; } @@ -79,21 +76,20 @@ namespace DistTestCore.Codex var nodePeerId = debugInfo.id; var nodeName = Container.Name; - log.AddStringReplace(nodePeerId, nodeName); - log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName); + lifecycle.Log.AddStringReplace(nodePeerId, nodeName); + lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName); } catch (Exception e) { - log.Error($"Failed to start codex node: {e}. Test infra failure."); + lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure."); throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e); } } private Http Http(TimeSpan? timeoutOverride = null) { - var ip = Container.Pod.Cluster.HostAddress; - var port = Container.ServicePorts[0].Number; - return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1", timeoutOverride); + var address = lifecycle.Configuration.GetAddress(Container); + return new Http(lifecycle.Log, lifecycle.TimeSet, address, baseUrl: "/api/codex/v1", timeoutOverride); } } diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index dfaf3a1..a723baa 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -1,5 +1,4 @@ -using System.Runtime.InteropServices; -using DistTestCore.Marketplace; +using DistTestCore.Marketplace; using KubernetesWorkflow; namespace DistTestCore.Codex @@ -56,6 +55,7 @@ namespace DistTestCore.Codex AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}"); AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account); AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address); + AddEnvVar("PERSISTENCE", "1"); } } } diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index 2005410..77d46e6 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -69,7 +69,7 @@ namespace DistTestCore private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory) { - var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c); + var access = new CodexAccess(lifecycle, c); return factory.CreateOnlineCodexNode(access, this); } } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index ac7d470..b9f21de 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -34,5 +34,25 @@ namespace DistTestCore { return CodexLogLevel.Trace; } + + public TestRunnerLocation GetTestRunnerLocation() + { + return TestRunnerLocation.ExternalToCluster; + } + + public RunningContainerAddress GetAddress(RunningContainer container) + { + if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster) + { + return container.ClusterInternalAddress; + } + return container.ClusterExternalAddress; + } + } + + public enum TestRunnerLocation + { + ExternalToCluster, + InternalToCluster, } } diff --git a/DistTestCore/GethStarter.cs b/DistTestCore/GethStarter.cs index 3ac8ce0..92af53f 100644 --- a/DistTestCore/GethStarter.cs +++ b/DistTestCore/GethStarter.cs @@ -33,7 +33,7 @@ namespace DistTestCore private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode) { - var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log); + var interaction = marketplaceNetwork.StartInteraction(lifecycle); var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress; var accounts = companionNode.Accounts.Select(a => a.Account).ToArray(); @@ -52,7 +52,7 @@ namespace DistTestCore private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork) { - return new GethMarketplaceAccessFactory(lifecycle.Log, marketplaceNetwork); + return new GethMarketplaceAccessFactory(lifecycle, marketplaceNetwork); } private GethCompanionNodeInfo StartCompanionNode(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork) diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index b7c7481..30bf48c 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -1,4 +1,5 @@ -using Logging; +using KubernetesWorkflow; +using Logging; using Newtonsoft.Json; using System.Net.Http.Headers; using System.Net.Http.Json; @@ -10,17 +11,15 @@ namespace DistTestCore { private readonly BaseLog log; private readonly ITimeSet timeSet; - private readonly string host; - private readonly int port; + private readonly RunningContainerAddress address; private readonly string baseUrl; private readonly TimeSpan? timeoutOverride; - public Http(BaseLog log, ITimeSet timeSet, string host, int port, string baseUrl, TimeSpan? timeoutOverride = null) + public Http(BaseLog log, ITimeSet timeSet, RunningContainerAddress address, string baseUrl, TimeSpan? timeoutOverride = null) { this.log = log; this.timeSet = timeSet; - this.host = host; - this.port = port; + this.address = address; this.baseUrl = baseUrl; this.timeoutOverride = timeoutOverride; if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; @@ -110,7 +109,7 @@ namespace DistTestCore private string GetUrl() { - return $"{host}:{port}{baseUrl}"; + return $"{address.Host}:{address.Port}{baseUrl}"; } private void Log(string url, string message) diff --git a/DistTestCore/Marketplace/CodexContractsStarter.cs b/DistTestCore/Marketplace/CodexContractsStarter.cs index 412c25a..2571ef1 100644 --- a/DistTestCore/Marketplace/CodexContractsStarter.cs +++ b/DistTestCore/Marketplace/CodexContractsStarter.cs @@ -34,7 +34,7 @@ namespace DistTestCore.Marketplace var marketplaceAddress = extractor.ExtractMarketplaceAddress(); var abi = extractor.ExtractMarketplaceAbi(); - var interaction = bootstrapNode.StartInteraction(lifecycle.Log); + var interaction = bootstrapNode.StartInteraction(lifecycle); var tokenAddress = interaction.GetTokenAddress(marketplaceAddress); LogEnd("Contracts deployed."); diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index ab38cb1..6825ba5 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -1,5 +1,4 @@ using KubernetesWorkflow; -using Logging; using NethereumWorkflow; namespace DistTestCore.Marketplace @@ -21,13 +20,12 @@ namespace DistTestCore.Marketplace public string PubKey { get; } public Port DiscoveryPort { get; } - public NethereumInteraction StartInteraction(BaseLog log) + public NethereumInteraction StartInteraction(TestLifecycle lifecycle) { - var ip = RunningContainers.RunningPod.Cluster.HostAddress; - var port = RunningContainers.Containers[0].ServicePorts[0].Number; + var address = lifecycle.Configuration.GetAddress(RunningContainers.Containers[0]); var account = Account; - var creator = new NethereumInteractionCreator(log, ip, port, account.PrivateKey); + var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, account.PrivateKey); return creator.CreateWorkflow(); } } diff --git a/DistTestCore/Marketplace/MarketplaceAccess.cs b/DistTestCore/Marketplace/MarketplaceAccess.cs index 1cc19e7..33f68e6 100644 --- a/DistTestCore/Marketplace/MarketplaceAccess.cs +++ b/DistTestCore/Marketplace/MarketplaceAccess.cs @@ -17,14 +17,14 @@ namespace DistTestCore.Marketplace public class MarketplaceAccess : IMarketplaceAccess { - private readonly TestLog log; + private readonly TestLifecycle lifecycle; private readonly MarketplaceNetwork marketplaceNetwork; private readonly GethAccount account; private readonly CodexAccess codexAccess; - public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess) + public MarketplaceAccess(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess) { - this.log = log; + this.lifecycle = lifecycle; this.marketplaceNetwork = marketplaceNetwork; this.account = account; this.codexAccess = codexAccess; @@ -103,7 +103,7 @@ namespace DistTestCore.Marketplace public TestToken GetBalance() { - var interaction = marketplaceNetwork.StartInteraction(log); + var interaction = marketplaceNetwork.StartInteraction(lifecycle); var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account.Account); var balance = new TestToken(amount); @@ -114,7 +114,7 @@ namespace DistTestCore.Marketplace private void Log(string msg) { - log.Log($"{codexAccess.Container.Name} {msg}"); + lifecycle.Log.Log($"{codexAccess.Container.Name} {msg}"); } } diff --git a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs index cd37d81..5ad52eb 100644 --- a/DistTestCore/Marketplace/MarketplaceAccessFactory.cs +++ b/DistTestCore/Marketplace/MarketplaceAccessFactory.cs @@ -18,19 +18,19 @@ namespace DistTestCore.Marketplace public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory { - private readonly TestLog log; + private readonly TestLifecycle lifecycle; private readonly MarketplaceNetwork marketplaceNetwork; - public GethMarketplaceAccessFactory(TestLog log, MarketplaceNetwork marketplaceNetwork) + public GethMarketplaceAccessFactory(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork) { - this.log = log; + this.lifecycle = lifecycle; this.marketplaceNetwork = marketplaceNetwork; } public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access) { var companionNode = GetGethCompanionNode(access); - return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access); + return new MarketplaceAccess(lifecycle, marketplaceNetwork, companionNode, access); } private GethAccount GetGethCompanionNode(CodexAccess access) diff --git a/DistTestCore/Marketplace/MarketplaceNetwork.cs b/DistTestCore/Marketplace/MarketplaceNetwork.cs index 8378444..bba80a2 100644 --- a/DistTestCore/Marketplace/MarketplaceNetwork.cs +++ b/DistTestCore/Marketplace/MarketplaceNetwork.cs @@ -1,5 +1,4 @@ -using Logging; -using NethereumWorkflow; +using NethereumWorkflow; namespace DistTestCore.Marketplace { @@ -14,9 +13,9 @@ namespace DistTestCore.Marketplace public GethBootstrapNodeInfo Bootstrap { get; } public MarketplaceInfo Marketplace { get; } - public NethereumInteraction StartInteraction(BaseLog log) + public NethereumInteraction StartInteraction(TestLifecycle lifecycle) { - return Bootstrap.StartInteraction(log); + return Bootstrap.StartInteraction(lifecycle); } } } diff --git a/DistTestCore/Metrics/MetricsAccessFactory.cs b/DistTestCore/Metrics/MetricsAccessFactory.cs index 24103ab..18dae04 100644 --- a/DistTestCore/Metrics/MetricsAccessFactory.cs +++ b/DistTestCore/Metrics/MetricsAccessFactory.cs @@ -28,7 +28,7 @@ namespace DistTestCore.Metrics public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer) { - var query = new MetricsQuery(lifecycle.Log, lifecycle.TimeSet, prometheusContainer); + var query = new MetricsQuery(lifecycle, prometheusContainer); return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer); } } diff --git a/DistTestCore/Metrics/MetricsQuery.cs b/DistTestCore/Metrics/MetricsQuery.cs index f162690..fc19867 100644 --- a/DistTestCore/Metrics/MetricsQuery.cs +++ b/DistTestCore/Metrics/MetricsQuery.cs @@ -1,6 +1,5 @@ using DistTestCore.Codex; using KubernetesWorkflow; -using Logging; using System.Globalization; namespace DistTestCore.Metrics @@ -9,15 +8,16 @@ namespace DistTestCore.Metrics { private readonly Http http; - public MetricsQuery(BaseLog log, ITimeSet timeSet, RunningContainers runningContainers) + public MetricsQuery(TestLifecycle lifecycle, RunningContainers runningContainers) { RunningContainers = runningContainers; + var address = lifecycle.Configuration.GetAddress(runningContainers.Containers[0]); + http = new Http( - log, - timeSet, - runningContainers.RunningPod.Cluster.HostAddress, - runningContainers.Containers[0].ServicePorts[0].Number, + lifecycle.Log, + lifecycle.TimeSet, + address, "api/v1"); } diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 1505745..ffd34b1 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -13,6 +13,7 @@ namespace DistTestCore public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet) { Log = log; + Configuration = configuration; TimeSet = timeSet; workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)); @@ -24,6 +25,7 @@ namespace DistTestCore } public TestLog Log { get; } + public Configuration Configuration { get; } public ITimeSet TimeSet { get; } public FileManager FileManager { get; } public CodexStarter CodexStarter { get; } diff --git a/KubernetesWorkflow/RunningContainers.cs b/KubernetesWorkflow/RunningContainers.cs index 616056c..0b7b3fd 100644 --- a/KubernetesWorkflow/RunningContainers.cs +++ b/KubernetesWorkflow/RunningContainers.cs @@ -21,18 +21,22 @@ public class RunningContainer { - public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig) + public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig, RunningContainerAddress clusterExternalAddress, RunningContainerAddress clusterInternalAddress) { Pod = pod; Recipe = recipe; ServicePorts = servicePorts; Name = GetContainerName(recipe, startupConfig); + ClusterExternalAddress = clusterExternalAddress; + ClusterInternalAddress = clusterInternalAddress; } public string Name { get; } public RunningPod Pod { get; } public ContainerRecipe Recipe { get; } public Port[] ServicePorts { get; } + public RunningContainerAddress ClusterExternalAddress { get; } + public RunningContainerAddress ClusterInternalAddress { get; } private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig) { @@ -46,4 +50,16 @@ } } } + + public class RunningContainerAddress + { + public RunningContainerAddress(string host, int port) + { + Host = host; + Port = port; + } + + public string Host { get; } + public int Port { get; } + } } diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 20cea0f..f5546c5 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -80,10 +80,36 @@ namespace KubernetesWorkflow var servicePorts = runningPod.GetServicePortsForContainerRecipe(r); log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}"); - return new RunningContainer(runningPod, r, servicePorts, startupConfig); + return new RunningContainer(runningPod, r, servicePorts, startupConfig, + GetContainerExternalAddress(runningPod, servicePorts), + GetContainerInternalAddress(servicePorts)); + }).ToArray(); } + private RunningContainerAddress GetContainerExternalAddress(RunningPod pod, Port[] servicePorts) + { + return new RunningContainerAddress( + pod.Cluster.HostAddress, + GetServicePort(servicePorts)); + } + + private RunningContainerAddress GetContainerInternalAddress(Port[] servicePorts) + { + var serviceName = "service-" + numberSource.WorkflowNumber; + var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace; + + return new RunningContainerAddress( + $"http://{serviceName}.{namespaceName}.svc.cluster.local", + GetServicePort(servicePorts)); + } + + private static int GetServicePort(Port[] servicePorts) + { + if (servicePorts.Any()) return servicePorts.First().Number; + return 0; + } + private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig) { log.Debug(); diff --git a/Nethereum/NethereumInteraction.cs b/Nethereum/NethereumInteraction.cs index 46d6da0..624f20c 100644 --- a/Nethereum/NethereumInteraction.cs +++ b/Nethereum/NethereumInteraction.cs @@ -131,7 +131,7 @@ namespace NethereumWorkflow public class MintTokensFunction : FunctionMessage { [Parameter("address", "holder", 1)] - public string Holder { get; set; } + public string Holder { get; set; } = string.Empty; [Parameter("uint256", "amount", 2)] public BigInteger Amount { get; set; } @@ -141,6 +141,6 @@ namespace NethereumWorkflow public class GetTokenBalanceFunction : FunctionMessage { [Parameter("address", "owner", 1)] - public string Owner { get; set; } + public string Owner { get; set; } = string.Empty; } } diff --git a/Nethereum/NethereumInteractionCreator.cs b/Nethereum/NethereumInteractionCreator.cs index 3ac0431..ab5449c 100644 --- a/Nethereum/NethereumInteractionCreator.cs +++ b/Nethereum/NethereumInteractionCreator.cs @@ -26,7 +26,7 @@ namespace NethereumWorkflow private Web3 CreateWeb3() { var account = new Nethereum.Web3.Accounts.Account(privateKey); - return new Web3(account, $"http://{ip}:{port}"); + return new Web3(account, $"{ip}:{port}"); } } } diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 3a4884c..976ccc3 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -1,5 +1,4 @@ using DistTestCore; -using DistTestCore.Codex; using NUnit.Framework; using Utils;