Sets up support for internal and external container addresses

DistTestCore/Codex/CodexAccess.cs

@@ -1,17 +1,14 @@
 using KubernetesWorkflow;
-using Logging;
 
 namespace DistTestCore.Codex
 {
     public class CodexAccess
     {
-        private readonly BaseLog log;
+        private readonly TestLifecycle lifecycle;
-        private readonly ITimeSet timeSet;
 
-        public CodexAccess(BaseLog log, ITimeSet timeSet, RunningContainer runningContainer)
+        public CodexAccess(TestLifecycle lifecycle, RunningContainer runningContainer)
         {
-            this.log = log;
+            this.lifecycle = lifecycle;
-            this.timeSet = timeSet;
             Container = runningContainer;
         }
 
@@ -79,21 +76,20 @@ namespace DistTestCore.Codex
 
                 var nodePeerId = debugInfo.id;
                 var nodeName = Container.Name;
-                log.AddStringReplace(nodePeerId, nodeName);
+                lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
-                log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
+                lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
             }
             catch (Exception e)
             {
-                log.Error($"Failed to start codex node: {e}. Test infra failure.");
+                lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure.");
                 throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
             }
         }
 
         private Http Http(TimeSpan? timeoutOverride = null)
         {
-            var ip = Container.Pod.Cluster.HostAddress;
+            var address = lifecycle.Configuration.GetAddress(Container);
-            var port = Container.ServicePorts[0].Number;
+            return new Http(lifecycle.Log, lifecycle.TimeSet, address, baseUrl: "/api/codex/v1", timeoutOverride);
-            return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1", timeoutOverride);
         }
     }
 

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -1,5 +1,4 @@
-using System.Runtime.InteropServices;
+using DistTestCore.Marketplace;
-using DistTestCore.Marketplace;
 using KubernetesWorkflow;
 
 namespace DistTestCore.Codex
@@ -56,6 +55,7 @@ namespace DistTestCore.Codex
                 AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}");
                 AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account);
                 AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
+                AddEnvVar("PERSISTENCE", "1");
             }
         }
     }

DistTestCore/CodexNodeGroup.cs

@@ -69,7 +69,7 @@ namespace DistTestCore
 
         private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
         {
-            var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c);
+            var access = new CodexAccess(lifecycle, c);
             return factory.CreateOnlineCodexNode(access, this);
         }
     }

DistTestCore/Configuration.cs

@@ -34,5 +34,25 @@ namespace DistTestCore
         {
             return CodexLogLevel.Trace;
         }
+
+        public TestRunnerLocation GetTestRunnerLocation()
+        {
+            return TestRunnerLocation.ExternalToCluster;
+        }
+
+        public RunningContainerAddress GetAddress(RunningContainer container)
+        {
+            if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster)
+            {
+                return container.ClusterInternalAddress;
+            }
+            return container.ClusterExternalAddress;
+        }
+    }
+
+    public enum TestRunnerLocation
+    {
+        ExternalToCluster,
+        InternalToCluster,
     }
 }

DistTestCore/GethStarter.cs

@@ -33,7 +33,7 @@ namespace DistTestCore
 
         private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
         {
-            var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log);
+            var interaction = marketplaceNetwork.StartInteraction(lifecycle);
             var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
 
             var accounts = companionNode.Accounts.Select(a => a.Account).ToArray();
@@ -52,7 +52,7 @@ namespace DistTestCore
 
         private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork)
         {
-            return new GethMarketplaceAccessFactory(lifecycle.Log, marketplaceNetwork);
+            return new GethMarketplaceAccessFactory(lifecycle, marketplaceNetwork);
         }
 
         private GethCompanionNodeInfo StartCompanionNode(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)

DistTestCore/Http.cs

@@ -1,4 +1,5 @@
-using Logging;
+using KubernetesWorkflow;
+using Logging;
 using Newtonsoft.Json;
 using System.Net.Http.Headers;
 using System.Net.Http.Json;
@@ -10,17 +11,15 @@ namespace DistTestCore
     {
         private readonly BaseLog log;
         private readonly ITimeSet timeSet;
-        private readonly string host;
+        private readonly RunningContainerAddress address;
-        private readonly int port;
         private readonly string baseUrl;
         private readonly TimeSpan? timeoutOverride;
 
-        public Http(BaseLog log, ITimeSet timeSet, string host, int port, string baseUrl, TimeSpan? timeoutOverride = null)
+        public Http(BaseLog log, ITimeSet timeSet, RunningContainerAddress address, string baseUrl, TimeSpan? timeoutOverride = null)
         {
             this.log = log;
             this.timeSet = timeSet;
-            this.host = host;
+            this.address = address;
-            this.port = port;
             this.baseUrl = baseUrl;
             this.timeoutOverride = timeoutOverride;
             if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
@@ -110,7 +109,7 @@ namespace DistTestCore
 
         private string GetUrl()
         {
-            return $"{host}:{port}{baseUrl}";
+            return $"{address.Host}:{address.Port}{baseUrl}";
         }
 
         private void Log(string url, string message)

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -34,7 +34,7 @@ namespace DistTestCore.Marketplace
             var marketplaceAddress = extractor.ExtractMarketplaceAddress();
             var abi = extractor.ExtractMarketplaceAbi();
 
-            var interaction = bootstrapNode.StartInteraction(lifecycle.Log);
+            var interaction = bootstrapNode.StartInteraction(lifecycle);
             var tokenAddress = interaction.GetTokenAddress(marketplaceAddress);
 
             LogEnd("Contracts deployed.");

DistTestCore/Marketplace/GethBootstrapNodeInfo.cs

@@ -1,5 +1,4 @@
 using KubernetesWorkflow;
-using Logging;
 using NethereumWorkflow;
 
 namespace DistTestCore.Marketplace
@@ -21,13 +20,12 @@ namespace DistTestCore.Marketplace
         public string PubKey { get; }
         public Port DiscoveryPort { get; }
 
-        public NethereumInteraction StartInteraction(BaseLog log)
+        public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
         {
-            var ip = RunningContainers.RunningPod.Cluster.HostAddress;
+            var address = lifecycle.Configuration.GetAddress(RunningContainers.Containers[0]);
-            var port = RunningContainers.Containers[0].ServicePorts[0].Number;
             var account = Account;
 
-            var creator = new NethereumInteractionCreator(log, ip, port, account.PrivateKey);
+            var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, account.PrivateKey);
             return creator.CreateWorkflow();
         }
     }

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -17,14 +17,14 @@ namespace DistTestCore.Marketplace
 
     public class MarketplaceAccess : IMarketplaceAccess
     {
-        private readonly TestLog log;
+        private readonly TestLifecycle lifecycle;
         private readonly MarketplaceNetwork marketplaceNetwork;
         private readonly GethAccount account;
         private readonly CodexAccess codexAccess;
 
-        public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
+        public MarketplaceAccess(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
         {
-            this.log = log;
+            this.lifecycle = lifecycle;
             this.marketplaceNetwork = marketplaceNetwork;
             this.account = account;
             this.codexAccess = codexAccess;
@@ -103,7 +103,7 @@ namespace DistTestCore.Marketplace
 
         public TestToken GetBalance()
         {
-            var interaction = marketplaceNetwork.StartInteraction(log);
+            var interaction = marketplaceNetwork.StartInteraction(lifecycle);
             var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account.Account);
             var balance = new TestToken(amount);
 
@@ -114,7 +114,7 @@ namespace DistTestCore.Marketplace
 
         private void Log(string msg)
         {
-            log.Log($"{codexAccess.Container.Name} {msg}");
+            lifecycle.Log.Log($"{codexAccess.Container.Name} {msg}");
         }
     }
 

DistTestCore/Marketplace/MarketplaceAccessFactory.cs

@@ -18,19 +18,19 @@ namespace DistTestCore.Marketplace
 
     public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory
     {
-        private readonly TestLog log;
+        private readonly TestLifecycle lifecycle;
         private readonly MarketplaceNetwork marketplaceNetwork;
 
-        public GethMarketplaceAccessFactory(TestLog log, MarketplaceNetwork marketplaceNetwork)
+        public GethMarketplaceAccessFactory(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork)
         {
-            this.log = log;
+            this.lifecycle = lifecycle;
             this.marketplaceNetwork = marketplaceNetwork;
         }
 
         public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
         {
             var companionNode = GetGethCompanionNode(access);
-            return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access);
+            return new MarketplaceAccess(lifecycle, marketplaceNetwork, companionNode, access);
         }
 
         private GethAccount GetGethCompanionNode(CodexAccess access)

DistTestCore/Marketplace/MarketplaceNetwork.cs

@@ -1,5 +1,4 @@
-using Logging;
+using NethereumWorkflow;
-using NethereumWorkflow;
 
 namespace DistTestCore.Marketplace
 {
@@ -14,9 +13,9 @@ namespace DistTestCore.Marketplace
         public GethBootstrapNodeInfo Bootstrap { get; }
         public MarketplaceInfo Marketplace { get; }
 
-        public NethereumInteraction StartInteraction(BaseLog log)
+        public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
         {
-            return Bootstrap.StartInteraction(log);
+            return Bootstrap.StartInteraction(lifecycle);
         }
     }
 }

DistTestCore/Metrics/MetricsAccessFactory.cs

@@ -28,7 +28,7 @@ namespace DistTestCore.Metrics
 
         public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
         {
-            var query = new MetricsQuery(lifecycle.Log, lifecycle.TimeSet, prometheusContainer);
+            var query = new MetricsQuery(lifecycle, prometheusContainer);
             return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
         }
     }

DistTestCore/Metrics/MetricsQuery.cs

@@ -1,6 +1,5 @@
 using DistTestCore.Codex;
 using KubernetesWorkflow;
-using Logging;
 using System.Globalization;
 
 namespace DistTestCore.Metrics
@@ -9,15 +8,16 @@ namespace DistTestCore.Metrics
     {
         private readonly Http http;
 
-        public MetricsQuery(BaseLog log, ITimeSet timeSet, RunningContainers runningContainers)
+        public MetricsQuery(TestLifecycle lifecycle, RunningContainers runningContainers)
         {
             RunningContainers = runningContainers;
 
+            var address = lifecycle.Configuration.GetAddress(runningContainers.Containers[0]);
+
             http = new Http(
-                log,
+                lifecycle.Log,
-                timeSet,
+                lifecycle.TimeSet,
-                runningContainers.RunningPod.Cluster.HostAddress,
+                address,
-                runningContainers.Containers[0].ServicePorts[0].Number,
                 "api/v1");
         }
 

DistTestCore/TestLifecycle.cs

@@ -13,6 +13,7 @@ namespace DistTestCore
         public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
         {
             Log = log;
+            Configuration = configuration;
             TimeSet = timeSet;
             workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet));
 
@@ -24,6 +25,7 @@ namespace DistTestCore
         }
 
         public TestLog Log { get; }
+        public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
         public FileManager FileManager { get; }
         public CodexStarter CodexStarter { get; }

KubernetesWorkflow/RunningContainers.cs

@@ -21,18 +21,22 @@
 
     public class RunningContainer
     {
-        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig)
+        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig, RunningContainerAddress clusterExternalAddress, RunningContainerAddress clusterInternalAddress)
         {
             Pod = pod;
             Recipe = recipe;
             ServicePorts = servicePorts;
             Name = GetContainerName(recipe, startupConfig);
+            ClusterExternalAddress = clusterExternalAddress;
+            ClusterInternalAddress = clusterInternalAddress;
         }
 
         public string Name { get; }
         public RunningPod Pod { get; }
         public ContainerRecipe Recipe { get; }
         public Port[] ServicePorts { get; }
+        public RunningContainerAddress ClusterExternalAddress { get; }
+        public RunningContainerAddress ClusterInternalAddress { get; }
 
         private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
         {
@@ -46,4 +50,16 @@
             }
         }
     }
+
+    public class RunningContainerAddress
+    {
+        public RunningContainerAddress(string host, int port)
+        {
+            Host = host;
+            Port = port;
+        }
+
+        public string Host { get; }
+        public int Port { get; }
+    }
 }

KubernetesWorkflow/StartupWorkflow.cs

@@ -80,10 +80,36 @@ namespace KubernetesWorkflow
                 var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
                 log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
 
-                return new RunningContainer(runningPod, r, servicePorts, startupConfig);
+                return new RunningContainer(runningPod, r, servicePorts, startupConfig,
+                    GetContainerExternalAddress(runningPod, servicePorts),
+                    GetContainerInternalAddress(servicePorts));
+
             }).ToArray();
         }
 
+        private RunningContainerAddress GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
+        {
+            return new RunningContainerAddress(
+                pod.Cluster.HostAddress,
+                GetServicePort(servicePorts));
+        }
+
+        private RunningContainerAddress GetContainerInternalAddress(Port[] servicePorts)
+        {
+            var serviceName = "service-" + numberSource.WorkflowNumber;
+            var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace;
+
+            return new RunningContainerAddress(
+                $"http://{serviceName}.{namespaceName}.svc.cluster.local",
+                GetServicePort(servicePorts));
+        }
+
+        private static int GetServicePort(Port[] servicePorts)
+        {
+            if (servicePorts.Any()) return servicePorts.First().Number;
+            return 0;
+        }
+
         private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             log.Debug();

Nethereum/NethereumInteraction.cs

@@ -131,7 +131,7 @@ namespace NethereumWorkflow
     public class MintTokensFunction : FunctionMessage
     {
         [Parameter("address", "holder", 1)]
-        public string Holder { get; set; }
+        public string Holder { get; set; } = string.Empty;
 
         [Parameter("uint256", "amount", 2)]
         public BigInteger Amount { get; set; }
@@ -141,6 +141,6 @@ namespace NethereumWorkflow
     public class GetTokenBalanceFunction : FunctionMessage
     {
         [Parameter("address", "owner", 1)]
-        public string Owner { get; set; }
+        public string Owner { get; set; } = string.Empty;
     }
 }

Nethereum/NethereumInteractionCreator.cs

@@ -26,7 +26,7 @@ namespace NethereumWorkflow
         private Web3 CreateWeb3()
         {
             var account = new Nethereum.Web3.Accounts.Account(privateKey);
-            return new Web3(account, $"http://{ip}:{port}");
+            return new Web3(account, $"{ip}:{port}");
         }
     }
 }

Tests/BasicTests/ExampleTests.cs

@@ -1,5 +1,4 @@
 using DistTestCore;
-using DistTestCore.Codex;
 using NUnit.Framework;
 using Utils;