Merge branch 'master' into feature/tests

.github/workflows/dist-tests.yaml

@@ -0,0 +1,73 @@
+name: Dist Tests
+
+
+on:
+  # push:
+  #   branches:
+  #     - master
+  #   tags:
+  #     - 'v*.*.*'
+  #   paths-ignore:
+  #     - '**/*.md'
+  #     - '.gitignore'
+  #     - 'docker/**'
+  #     - '!docker/job.yaml'
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: Branch
+        required: false
+        type: string
+      source:
+        description: Repository with tests
+        required: false
+        type: string
+      nameprefix:
+        description: Runner job/pod name prefix
+        required: false
+        type: string
+      namespace:
+        description: Kubernetes namespace for runner
+        required: false
+        type: string
+
+
+env:
+  BRANCH: ${{ github.ref_name }}
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  NAMEPREFIX: cs-codex-dist-tests
+  NAMESPACE: cs-codex-dist-tests
+  JOB_MANIFEST: docker/job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.26.1
+
+
+jobs:
+  run_tests:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Variables
+        run: |
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v3
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job
+        run: |
+          export RUNID=$(date +%Y%m%d-%H%M%S)
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -

DistTestCore/AutoBootstrapDistTest.cs

@@ -1,9 +1,9 @@
-namespace DistTestCore
+using NUnit.Framework;
+
+namespace DistTestCore
 {
     public class AutoBootstrapDistTest : DistTest
     {
-        private IOnlineCodexNode? bootstrapNode;
-
         public override IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
         {
             throw new Exception("AutoBootstrapDistTest creates and attaches a single boostrap node for you. " +
@@ -12,19 +12,18 @@
 
         public override ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
         {
-            var codexSetup = new CodexSetup(numberOfNodes);
+            var codexSetup = CreateCodexSetup(numberOfNodes);
             setup(codexSetup);
-            codexSetup.WithBootstrapNode(EnsureBootstapNode());
+            codexSetup.WithBootstrapNode(BootstrapNode);
             return BringOnline(codexSetup);
         }
 
-        private IOnlineCodexNode EnsureBootstapNode()
+        [SetUp]
+        public void SetUpBootstrapNode()
         {
-            if (bootstrapNode == null)
+            BootstrapNode = BringOnline(CreateCodexSetup(1))[0];
-            {
-                bootstrapNode = base.SetupCodexBootstrapNode(s => { });
-            }
-            return bootstrapNode;
         }
+
+        protected IOnlineCodexNode BootstrapNode { get; private set; } = null!;
     }
 }

DistTestCore/BaseStarter.cs

@@ -32,6 +32,11 @@ namespace DistTestCore
             lifecycle.Log.Log($"{GetClassName()} {msg}");
         }
 
+        protected void Debug(string msg)
+        {
+            lifecycle.Log.Debug($"{GetClassName()} {msg}", 1);
+        }
+
         private string GetClassName()
         {
             return $"({GetType().Name})";

DistTestCore/Codex/CodexAccess.cs

@@ -1,17 +1,14 @@
 using KubernetesWorkflow;
-using Logging;
 
 namespace DistTestCore.Codex
 {
     public class CodexAccess
     {
-        private readonly BaseLog log;
+        private readonly TestLifecycle lifecycle;
-        private readonly ITimeSet timeSet;
 
-        public CodexAccess(BaseLog log, ITimeSet timeSet, RunningContainer runningContainer)
+        public CodexAccess(TestLifecycle lifecycle, RunningContainer runningContainer)
         {
-            this.log = log;
+            this.lifecycle = lifecycle;
-            this.timeSet = timeSet;
             Container = runningContainer;
         }
 
@@ -19,7 +16,30 @@ namespace DistTestCore.Codex
 
         public CodexDebugResponse GetDebugInfo()
         {
-            return Http().HttpGetJson<CodexDebugResponse>("debug/info");
+            return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugResponse>("debug/info");
+        }
+
+        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        {
+            return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
+        }
+
+        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
+        {
+            var http = Http(timeout);
+            var str = http.HttpGetString($"debug/peer/{peerId}");
+
+            if (str.ToLowerInvariant() == "unable to find peer!")
+            {
+                return new CodexDebugPeerResponse
+                {
+                    IsPeerFound = false
+                };
+            }
+
+            var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
+            result.IsPeerFound = true;
+            return result;
         }
 
         public string UploadFile(FileStream fileStream)
@@ -42,6 +62,11 @@ namespace DistTestCore.Codex
             return Http().HttpPostJson($"storage/request/{contentId}", request);
         }
 
+        public string ConnectToPeer(string peerId, string peerMultiAddress)
+        {
+            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
+        }
+
         public void EnsureOnline()
         {
             try
@@ -51,25 +76,20 @@ namespace DistTestCore.Codex
 
                 var nodePeerId = debugInfo.id;
                 var nodeName = Container.Name;
-                log.AddStringReplace(nodePeerId, $"___{nodeName}___");
+                lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
+                lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
             }
             catch (Exception e)
             {
-                log.Error($"Failed to start codex node: {e}. Test infra failure.");
+                lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure.");
                 throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
             }
         }
 
-        private Http Http()
+        private Http Http(TimeSpan? timeoutOverride = null)
         {
-            var ip = Container.Pod.Cluster.IP;
+            var address = lifecycle.Configuration.GetAddress(Container);
-            var port = Container.ServicePorts[0].Number;
+            return new Http(lifecycle.Log, lifecycle.TimeSet, address, baseUrl: "/api/codex/v1", timeoutOverride);
-            return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1");
-        }
-
-        public string ConnectToPeer(string peerId, string peerMultiAddress)
-        {
-            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
         }
     }
 
@@ -82,6 +102,22 @@ namespace DistTestCore.Codex
         public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
         public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
         public CodexDebugVersionResponse codex { get; set; } = new();
+        public CodexDebugTableResponse table { get; set; } = new();
+    }
+
+    public class CodexDebugTableResponse
+    {
+        public CodexDebugTableNodeResponse localNode { get; set; } = new();
+        public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
+    }
+
+    public class CodexDebugTableNodeResponse
+    {
+        public string nodeId { get; set; } = string.Empty;
+        public string peerId { get; set; } = string.Empty;
+        public string record { get; set; } = string.Empty;
+        public string address { get; set; } = string.Empty;
+        public bool seen { get; set; }
     }
 
     public class EnginePeerResponse
@@ -110,6 +146,20 @@ namespace DistTestCore.Codex
         public string revision { get; set; } = string.Empty;
     }
 
+    public class CodexDebugPeerResponse
+    {
+        public bool IsPeerFound { get; set; }
+
+        public string peerId { get; set; } = string.Empty;
+        public long seqNo { get; set; }
+        public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
+    }
+
+    public class CodexDebugPeerAddressResponse
+    {
+        public string address { get; set; } = string.Empty;
+    }
+
     public class CodexSalesAvailabilityRequest
     {
         public string size { get; set; } = string.Empty;

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -1,5 +1,4 @@
-using System.Runtime.InteropServices;
+using DistTestCore.Marketplace;
-using DistTestCore.Marketplace;
 using KubernetesWorkflow;
 
 namespace DistTestCore.Codex
@@ -7,12 +6,13 @@ namespace DistTestCore.Codex
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
         #if Arm64
-            public const string DockerImage = "emizzle/nim-codex-arm64:sha-c7af585";
+            public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0";
         #else
-			//public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635";
+			//public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0";
-            public const string DockerImage = "thatbenbierens/codexlocal:latest";
+            public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0";
         #endif
         public const string MetricsPortTag = "metrics_port";
+        public const string DiscoveryPortTag = "discovery-port";
 
         protected override string Image => DockerImage;
 
@@ -22,7 +22,8 @@ namespace DistTestCore.Codex
 
             AddExposedPortAndVar("API_PORT");
             AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}");
-            AddInternalPortAndVar("DISC_PORT");
+            AddInternalPortAndVar("DISC_PORT", DiscoveryPortTag);
+            AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
 
             var listenPort = AddInternalPort();
             AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
@@ -31,11 +32,6 @@ namespace DistTestCore.Codex
             {
                 AddEnvVar("BOOTSTRAP_SPR", config.BootstrapSpr);
             }
-
-            if (config.LogLevel != null)
-            {
-                AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
-            }
             if (config.StorageQuota != null)
             {
                 AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
@@ -53,12 +49,13 @@ namespace DistTestCore.Codex
                 var companionNodeAccount = companionNode.Accounts[Index];
                 Additional(companionNodeAccount);
 
-                var ip = companionNode.RunningContainer.Pod.Ip;
+                var ip = companionNode.RunningContainer.Pod.PodInfo.Ip;
                 var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number;
 
                 AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}");
                 AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account);
                 AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
+                AddEnvVar("PERSISTENCE", "1");
             }
         }
     }

DistTestCore/Codex/CodexStartupConfig.cs

@@ -5,9 +5,14 @@ namespace DistTestCore.Codex
 {
     public class CodexStartupConfig
     {
+        public CodexStartupConfig(CodexLogLevel logLevel)
+        {
+            LogLevel = logLevel;
+        }
+
         public string? NameOverride { get; set; }
         public Location Location { get; set; }
-        public CodexLogLevel? LogLevel { get; set; }
+        public CodexLogLevel LogLevel { get; }
         public ByteSize? StorageQuota { get; set; }
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }

DistTestCore/CodexNodeGroup.cs

@@ -69,7 +69,7 @@ namespace DistTestCore
 
         private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
         {
-            var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c);
+            var access = new CodexAccess(lifecycle, c);
             return factory.CreateOnlineCodexNode(access, this);
         }
     }

DistTestCore/CodexSetup.cs

@@ -8,7 +8,6 @@ namespace DistTestCore
     {
         ICodexSetup WithName(string name);
         ICodexSetup At(Location location);
-        ICodexSetup WithLogLevel(CodexLogLevel level);
         ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
         ICodexSetup WithStorageQuota(ByteSize storageQuota);
         ICodexSetup EnableMetrics();
@@ -20,7 +19,8 @@ namespace DistTestCore
     {
         public int NumberOfNodes { get; }
 
-        public CodexSetup(int numberOfNodes)
+        public CodexSetup(int numberOfNodes, CodexLogLevel logLevel)
+            : base(logLevel)
         {
             NumberOfNodes = numberOfNodes;
         }
@@ -43,12 +43,6 @@ namespace DistTestCore
             return this;
         }
 
-        public ICodexSetup WithLogLevel(CodexLogLevel level)
-        {
-            LogLevel = level;
-            return this;
-        }
-
         public ICodexSetup WithStorageQuota(ByteSize storageQuota)
         {
             StorageQuota = storageQuota;
@@ -80,7 +74,7 @@ namespace DistTestCore
 
         private IEnumerable<string> DescribeArgs()
         {
-            if (LogLevel != null) yield return $"LogLevel={LogLevel}";
+            yield return $"LogLevel={LogLevel}";
             if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
             if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
         }

DistTestCore/CodexStarter.cs

@@ -1,6 +1,7 @@
 using DistTestCore.Codex;
 using DistTestCore.Marketplace;
 using KubernetesWorkflow;
+using Logging;
 
 namespace DistTestCore
 {
@@ -27,7 +28,8 @@ namespace DistTestCore
             var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
 
             var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
-            LogEnd($"Started {codexSetup.NumberOfNodes} nodes at '{group.Containers.RunningPod.Ip}'. They are: {group.Describe()}");
+            var podInfo = group.Containers.RunningPod.PodInfo;
+            LogEnd($"Started {codexSetup.NumberOfNodes} nodes at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}");
             LogSeparator();
             return group;
         }
@@ -74,7 +76,7 @@ namespace DistTestCore
         {
             var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
             RunningGroups.Add(group);
-            group.EnsureOnline();
+            Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
             return group;
         }
 

DistTestCore/Configuration.cs

@@ -1,32 +1,89 @@
-using KubernetesWorkflow;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
 
 namespace DistTestCore
 {
     public class Configuration
     {
+        private readonly string? kubeConfigFile;
+        private readonly string logPath;
+        private readonly bool logDebug;
+        private readonly string dataFilesPath;
+        private readonly CodexLogLevel codexLogLevel;
+        private readonly TestRunnerLocation runnerLocation;
+
+        public Configuration()
+        {
+            kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
+            logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
+            logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
+            dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
+            codexLogLevel = ParseEnum<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
+            runnerLocation = ParseEnum<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
+        }
+
         public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
         {
             return new KubernetesWorkflow.Configuration(
                 k8sNamespacePrefix: "ct-",
-                kubeConfigFile: null,
+                kubeConfigFile: kubeConfigFile,
                 operationTimeout: timeSet.K8sOperationTimeout(),
-                retryDelay: timeSet.WaitForK8sServiceDelay(),
+                retryDelay: timeSet.WaitForK8sServiceDelay()
-                locationMap: new[]
-                {
-                    new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),
-                    new ConfigurationLocationEntry(Location.BensLaptop, "worker02"),
-                } 
             );
         }
 
         public Logging.LogConfig GetLogConfig()
         {
-            return new Logging.LogConfig("CodexTestLogs", debugEnabled: false);
+            return new Logging.LogConfig(logPath, debugEnabled: logDebug);
         }
 
         public string GetFileManagerFolder()
         {
-            return "TestDataFiles";
+            return dataFilesPath;
+        }
+
+        public CodexLogLevel GetCodexLogLevel()
+        {
+            return codexLogLevel;
+        }
+
+        public TestRunnerLocation GetTestRunnerLocation()
+        {
+            return runnerLocation;
+        }
+
+        public RunningContainerAddress GetAddress(RunningContainer container)
+        {
+            if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster)
+            {
+                return container.ClusterInternalAddress;
+            }
+            return container.ClusterExternalAddress;
+        }
+
+        private static string GetEnvVarOrDefault(string varName, string defaultValue)
+        {
+            var v = Environment.GetEnvironmentVariable(varName);
+            if (v == null) return defaultValue;
+            return v;
+        }
+
+        private static string? GetNullableEnvVarOrDefault(string varName, string? defaultValue)
+        {
+            var v = Environment.GetEnvironmentVariable(varName);
+            if (v == null) return defaultValue;
+            return v;
+        }
+
+        private static T ParseEnum<T>(string value)
+        {
+            return (T)Enum.Parse(typeof(T), value, true);
         }
     }
+
+    public enum TestRunnerLocation
+    {
+        ExternalToCluster,
+        InternalToCluster,
+    }
 }

DistTestCore/DistTest.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Codex;
+using DistTestCore.Helpers;
 using DistTestCore.Logs;
 using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
@@ -25,8 +26,14 @@ namespace DistTestCore
             testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
 
             fixtureLog = new FixtureLog(configuration.GetLogConfig());
+
+            PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
+            PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
         }
 
+        public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; }
+        public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; }
+
         [OneTimeSetUp]
         public void GlobalSetup()
         {
@@ -85,6 +92,17 @@ namespace DistTestCore
             return Get().FileManager.GenerateTestFile(size);
         }
 
+        /// <summary>
+        /// Any test files generated in 'action' will be deleted after it returns.
+        /// This helps prevent large tests from filling up discs.
+        /// </summary>
+        public void ScopedTestFiles(Action action)
+        {
+            Get().FileManager.PushFileSet();
+            action();
+            Get().FileManager.PopFileSet();
+        }
+
         public IOnlineCodexNode SetupCodexBootstrapNode()
         {
             return SetupCodexBootstrapNode(s => { });
@@ -116,7 +134,7 @@ namespace DistTestCore
 
         public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
         {
-            var codexSetup = new CodexSetup(numberOfNodes);
+            var codexSetup = CreateCodexSetup(numberOfNodes);
 
             setup(codexSetup);
 
@@ -128,16 +146,31 @@ namespace DistTestCore
             return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
         }
 
-        protected void Log(string msg)
+        public IEnumerable<IOnlineCodexNode> GetAllOnlineCodexNodes()
         {
-            TestContext.Progress.WriteLine(msg);
+            return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
-            Get().Log.Log(msg);
         }
 
-        protected void Debug(string msg)
+        public BaseLog GetTestLog()
+        {
+            return Get().Log;
+        }
+
+        public void Log(string msg)
         {
             TestContext.Progress.WriteLine(msg);
-            Get().Log.Debug(msg);
+            GetTestLog().Log(msg);
+        }
+
+        public void Debug(string msg)
+        {
+            TestContext.Progress.WriteLine(msg);
+            GetTestLog().Debug(msg);
+        }
+
+        protected CodexSetup CreateCodexSetup(int numberOfNodes)
+        {
+            return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
         }
 
         private TestLifecycle Get()

DistTestCore/FileManager.cs

@@ -9,6 +9,8 @@ namespace DistTestCore
         TestFile CreateEmptyTestFile();
         TestFile GenerateTestFile(ByteSize size);
         void DeleteAllTestFiles();
+        void PushFileSet();
+        void PopFileSet();
     }
 
     public class FileManager : IFileManager
@@ -18,6 +20,7 @@ namespace DistTestCore
         private readonly Random random = new Random();
         private readonly TestLog log;
         private readonly string folder;
+        private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();
 
         public FileManager(TestLog log, Configuration configuration)
         {
@@ -31,6 +34,7 @@ namespace DistTestCore
         {
             var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin"));
             File.Create(result.Filename).Close();
+            if (fileSetStack.Any()) fileSetStack.Last().Add(result);
             return result;
         }
 
@@ -47,6 +51,27 @@ namespace DistTestCore
             DeleteDirectory();
         }
 
+        public void PushFileSet()
+        {
+            fileSetStack.Add(new List<TestFile>());
+        }
+
+        public void PopFileSet()
+        {
+            if (!fileSetStack.Any()) return;
+            var pop = fileSetStack.Last();
+            fileSetStack.Remove(pop);
+
+            foreach (var file in pop)
+            {
+                try
+                {
+                    File.Delete(file.Filename);
+                }
+                catch { }
+            }
+        }
+
         private void GenerateFileBytes(TestFile result, ByteSize size)
         {
             long bytesLeft = size.SizeInBytes;

DistTestCore/GethStarter.cs

@@ -33,7 +33,7 @@ namespace DistTestCore
 
         private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
         {
-            var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log);
+            var interaction = marketplaceNetwork.StartInteraction(lifecycle);
             var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
 
             var accounts = companionNode.Accounts.Select(a => a.Account).ToArray();
@@ -52,7 +52,7 @@ namespace DistTestCore
 
         private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork)
         {
-            return new GethMarketplaceAccessFactory(lifecycle.Log, marketplaceNetwork);
+            return new GethMarketplaceAccessFactory(lifecycle, marketplaceNetwork);
         }
 
         private GethCompanionNodeInfo StartCompanionNode(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)

DistTestCore/Helpers/AssertHelpers.cs

@@ -0,0 +1,22 @@
+using NUnit.Framework.Constraints;
+using NUnit.Framework;
+using Utils;
+
+namespace DistTestCore.Helpers
+{
+    public static class AssertHelpers
+    {
+        public static void RetryAssert<T>(IResolveConstraint constraint, Func<T> actual, string message)
+        {
+            try
+            {
+                var c = constraint.Resolve();
+                Time.WaitUntil(() => c.ApplyTo(actual()).IsSuccess);
+            }
+            catch (TimeoutException)
+            {
+                Assert.That(actual(), constraint, message);
+            }
+        }
+    }
+}

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -0,0 +1,239 @@
+using DistTestCore.Codex;
+using NUnit.Framework;
+using Utils;
+
+namespace DistTestCore.Helpers
+{
+    public class PeerConnectionTestHelpers
+    {
+        private readonly Random random = new Random();
+        private readonly DistTest test;
+
+        public PeerConnectionTestHelpers(DistTest test)
+        {
+            this.test = test;
+        }
+
+        public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullyConnected(nodes.ToArray());
+        }
+
+        public void AssertFullyConnected(params IOnlineCodexNode[] nodes)
+        {
+            test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            var entries = CreateEntries(nodes);
+            var pairs = CreatePairs(entries);
+
+            RetryWhilePairs(pairs, () =>
+            {
+                CheckAndRemoveSuccessful(pairs);
+            });
+
+            if (pairs.Any())
+            {
+                test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
+                Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
+                test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
+            }
+            else
+            {
+                test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
+            }
+        }
+
+        private static void RetryWhilePairs(List<Pair> pairs, Action action)
+        {
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(10);
+            while (pairs.Any() && timeout > DateTime.UtcNow)
+            {
+                action();
+
+                if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(5));
+            }
+        }
+
+        private void CheckAndRemoveSuccessful(List<Pair> pairs)
+        {
+            var checkTasks = pairs.Select(p => Task.Run(() =>
+            {
+                ApplyRandomDelay();
+                p.Check();
+            })).ToArray();
+
+            Task.WaitAll(checkTasks);
+
+            foreach (var pair in pairs.ToArray())
+            {
+                if (pair.Success)
+                {
+                    test.Log(pair.GetMessage());
+                    pairs.Remove(pair);
+                }
+            }
+        }
+
+        private static Entry[] CreateEntries(IOnlineCodexNode[] nodes)
+        {
+            var entries = nodes.Select(n => new Entry(n)).ToArray();
+            var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
+
+            if (incorrectDiscoveryEndpoints.Any())
+            {
+                Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
+                    string.Join(Environment.NewLine, incorrectDiscoveryEndpoints));
+            }
+
+            return entries;
+        }
+
+        private static List<Pair> CreatePairs(Entry[] entries)
+        {
+            return CreatePairsIterator(entries).ToList();
+        }
+
+        private static IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
+        {
+            for (var x = 0; x < entries.Length; x++)
+            {
+                for (var y = x + 1; y < entries.Length; y++)
+                {
+                    yield return new Pair(entries[x], entries[y]);
+                }
+            }
+        }
+
+        private void ApplyRandomDelay()
+        {
+            // Calling all the nodes all at the same time is not exactly nice.
+            Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000)));
+        }
+
+        public class Entry
+        {
+            public Entry(IOnlineCodexNode node)
+            {
+                Node = node;
+                Response = node.GetDebugInfo();
+            }
+
+            public IOnlineCodexNode Node { get; }
+            public CodexDebugResponse Response { get; }
+
+            public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
+            {
+                foreach (var peer in Response.table.nodes)
+                {
+                    var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
+                    if (expected != peer.address)
+                    {
+                        yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'";
+                    }
+                }
+            }
+
+            private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
+            {
+                var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
+                if (peer == null) return $"peerId: {node.peerId} is not known.";
+
+                var n = (OnlineCodexNode)peer.Node;
+                var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
+                var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
+                return $"{ip}:{discPort.Number}";
+            }
+        }
+
+        public enum PeerConnectionState
+        {
+            Unknown,
+            Connection,
+            NoConnection,
+        }
+
+        public class Pair
+        {
+            private readonly TimeSpan timeout = TimeSpan.FromSeconds(60);
+            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
+            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
+
+            public Pair(Entry a, Entry b)
+            {
+                A = a;
+                B = b;
+            }
+
+            public Entry A { get; }
+            public Entry B { get; }
+            public PeerConnectionState AKnowsB { get; private set; }
+            public PeerConnectionState BKnowsA { get; private set; }
+            public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
+
+            public void Check()
+            {
+                aToBTime = Measure(() => AKnowsB = Knows(A, B));
+                bToATime = Measure(() => BKnowsA = Knows(B, A));
+            }
+
+            public string GetMessage()
+            {
+                return GetResultMessage() + GetTimePostfix();
+            }
+
+            private string GetResultMessage()
+            {
+                var aName = A.Response.id;
+                var bName = B.Response.id;
+
+                if (Success)
+                {
+                    return $"{aName} and {bName} know each other.";
+                }
+
+                return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}";
+            }
+
+            private string GetTimePostfix()
+            {
+                var aName = A.Response.id;
+                var bName = B.Response.id;
+
+                return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
+            }
+
+            private static TimeSpan Measure(Action action)
+            {
+                var start = DateTime.UtcNow;
+                action();
+                return DateTime.UtcNow - start;
+            }
+
+            private PeerConnectionState Knows(Entry a, Entry b)
+            {
+                lock (a)
+                {
+                    var peerId = b.Response.id;
+
+                    try
+                    {
+                        var response = a.Node.GetDebugPeer(peerId, timeout);
+                        if (!response.IsPeerFound)
+                        {
+                            return PeerConnectionState.NoConnection;
+                        }
+                        if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
+                        {
+                            return PeerConnectionState.Connection;
+                        }
+                    }
+                    catch
+                    {
+                    }
+
+                    // Didn't get a conclusive answer. Try again later.
+                    return PeerConnectionState.Unknown;
+                }
+            }
+        }
+    }
+}

DistTestCore/Helpers/PeerDownloadTestHelpers.cs

@@ -0,0 +1,52 @@
+namespace DistTestCore.Helpers
+{
+    public class PeerDownloadTestHelpers
+    {
+        private readonly DistTest test;
+
+        public PeerDownloadTestHelpers(DistTest test)
+        {
+            this.test = test;
+        }
+
+        public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullDownloadInterconnectivity(nodes, 1.MB());
+        }
+
+        public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
+        {
+            test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            foreach (var node in nodes)
+            {
+                var uploader = node;
+                var downloaders = nodes.Where(n => n != uploader).ToArray();
+
+                test.ScopedTestFiles(() =>
+                {
+                    PerformTest(uploader, downloaders, testFileSize);
+                });
+            }
+            
+            test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
+        }
+
+        private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
+        {
+            // 1 test file per downloader.
+            var files = downloaders.Select(d => test.GenerateTestFile(testFileSize)).ToArray();
+
+            // Upload all the test files to the uploader.
+            var contentIds = files.Select(uploader.UploadFile).ToArray();
+
+            // Each downloader should retrieve its own test file.
+            for (var i = 0; i < downloaders.Length; i++)
+            {
+                var expectedFile = files[i];
+                var downloadedFile = downloaders[i].DownloadContent(contentIds[i]);
+
+                expectedFile.AssertIsEqual(downloadedFile);
+            }
+        }
+    }
+}

DistTestCore/Http.cs

@@ -1,6 +1,6 @@
-using Logging;
+using KubernetesWorkflow;
+using Logging;
 using Newtonsoft.Json;
-using NUnit.Framework;
 using System.Net.Http.Headers;
 using System.Net.Http.Json;
 using Utils;
@@ -11,18 +11,17 @@ namespace DistTestCore
     {
         private readonly BaseLog log;
         private readonly ITimeSet timeSet;
-        private readonly string ip;
+        private readonly RunningContainerAddress address;
-        private readonly int port;
         private readonly string baseUrl;
+        private readonly TimeSpan? timeoutOverride;
 
-        public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl)
+        public Http(BaseLog log, ITimeSet timeSet, RunningContainerAddress address, string baseUrl, TimeSpan? timeoutOverride = null)
         {
             this.log = log;
             this.timeSet = timeSet;
-            this.ip = ip;
+            this.address = address;
-            this.port = port;
             this.baseUrl = baseUrl;
-
+            this.timeoutOverride = timeoutOverride;
             if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
             if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
         }
@@ -38,7 +37,7 @@ namespace DistTestCore
                 var str = Time.Wait(result.Content.ReadAsStringAsync());
                 Log(url, str);
                 return str; ;
-            });
+            }, $"HTTP-GET:{route}");
         }
 
         public T HttpGetJson<T>(string route)
@@ -62,10 +61,10 @@ namespace DistTestCore
                 using var content = JsonContent.Create(body);
                 Log(url, JsonConvert.SerializeObject(body));
                 var result = Time.Wait(client.PostAsync(url, content));
-                var str= Time.Wait(result.Content.ReadAsStringAsync());
+                var str = Time.Wait(result.Content.ReadAsStringAsync());
                 Log(url, str);
                 return str;
-            });
+            }, $"HTTP-POST-JSON: {route}");
         }
 
         public string HttpPostStream(string route, Stream stream)
@@ -81,7 +80,7 @@ namespace DistTestCore
                 var str =Time.Wait(response.Content.ReadAsStringAsync());
                 Log(url, str);
                 return str;
-            });
+            }, $"HTTP-POST-STREAM: {route}");
         }
 
         public Stream HttpGetStream(string route)
@@ -92,43 +91,10 @@ namespace DistTestCore
                 var url = GetUrl() + route;
                 Log(url, "~ STREAM ~");
                 return Time.Wait(client.GetStreamAsync(url));
-            });
+            }, $"HTTP-GET-STREAM: {route}");
         }
 
-        private string GetUrl()
+        public T TryJsonDeserialize<T>(string json)
-        {
-            return $"http://{ip}:{port}{baseUrl}";
-        }
-
-        private void Log(string url, string message)
-        {
-            log.Debug($"({url}) = '{message}'", 3);
-        }
-
-        private T Retry<T>(Func<T> operation)
-        {
-            var retryCounter = 0;
-
-            while (true)
-            {
-                try
-                {
-                    return operation();
-                }
-                catch (Exception exception)
-                {
-                    timeSet.HttpCallRetryDelay();
-                    retryCounter++;
-                    if (retryCounter > timeSet.HttpCallRetryCount())
-                    {
-                        Assert.Fail(exception.ToString());
-                        throw;
-                    }
-                }
-            }
-        }
-
-        private static T TryJsonDeserialize<T>(string json)
         {
             try
             {
@@ -137,15 +103,38 @@ namespace DistTestCore
             catch (Exception exception)
             {
                 var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}";
-                Assert.Fail(msg);
                 throw new InvalidOperationException(msg, exception);
             }
         }
 
+        private string GetUrl()
+        {
+            return $"{address.Host}:{address.Port}{baseUrl}";
+        }
+
+        private void Log(string url, string message)
+        {
+            log.Debug($"({url}) = '{message}'", 3);
+        }
+
+        private T Retry<T>(Func<T> operation, string description)
+        {
+            return Time.Retry(operation, timeSet.HttpCallRetryTimeout(), timeSet.HttpCallRetryDelay(), description);
+        }
+
         private HttpClient GetClient()
+        {
+            if (timeoutOverride.HasValue)
+            {
+                return GetClient(timeoutOverride.Value);
+            }
+            return GetClient(timeSet.HttpCallTimeout());
+        }
+
+        private HttpClient GetClient(TimeSpan timeout)
         {
             var client = new HttpClient();
-            client.Timeout = timeSet.HttpCallTimeout();
+            client.Timeout = timeout;
             return client;
         }
     }

DistTestCore/Marketplace/CodexContractsContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace DistTestCore.Marketplace
         #if Arm64
             public const string DockerImage = "emizzle/codex-contracts-deployment:latest";
         #else
-            public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint";
+            public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint2";
         #endif
         public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json";
         public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json";

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -5,7 +5,6 @@ namespace DistTestCore.Marketplace
 {
     public class CodexContractsStarter : BaseStarter
     {
-        private const string readyString = "Done! Sleeping indefinitely...";
 
         public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
             : base(lifecycle, workflowCreator)
@@ -14,7 +13,7 @@ namespace DistTestCore.Marketplace
 
         public MarketplaceInfo Start(GethBootstrapNodeInfo bootstrapNode)
         {
-            LogStart("Deploying Codex contracts...");
+            LogStart("Deploying Codex Marketplace...");
 
             var workflow = workflowCreator.CreateWorkflow();
             var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]);
@@ -25,32 +24,33 @@ namespace DistTestCore.Marketplace
 
             WaitUntil(() =>
             {
-                var logHandler = new ContractsReadyLogHandler(readyString);
+                var logHandler = new ContractsReadyLogHandler(Debug);
                 workflow.DownloadContainerLog(container, logHandler);
                 return logHandler.Found;
             });
+            Log("Contracts deployed. Extracting addresses...");
 
             var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
             var marketplaceAddress = extractor.ExtractMarketplaceAddress();
             var abi = extractor.ExtractMarketplaceAbi();
 
-            var interaction = bootstrapNode.StartInteraction(lifecycle.Log);
+            var interaction = bootstrapNode.StartInteraction(lifecycle);
             var tokenAddress = interaction.GetTokenAddress(marketplaceAddress);
 
-            LogEnd("Contracts deployed.");
+            LogEnd("Extract completed. Marketplace deployed.");
 
             return new MarketplaceInfo(marketplaceAddress, abi, tokenAddress);
         }
 
         private void WaitUntil(Func<bool> predicate)
         {
-            Time.WaitUntil(predicate, TimeSpan.FromMinutes(2), TimeSpan.FromSeconds(1));
+            Time.WaitUntil(predicate, TimeSpan.FromMinutes(3), TimeSpan.FromSeconds(2));
         }
 
         private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer)
         {
             var startupConfig = new StartupConfig();
-            var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
+            var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.PodInfo.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
             startupConfig.Add(contractsConfig);
             return startupConfig;
         }
@@ -72,18 +72,27 @@ namespace DistTestCore.Marketplace
 
     public class ContractsReadyLogHandler : LogHandler
     {
-        private readonly string targetString;
+        // Log should contain 'Compiled 15 Solidity files successfully' at some point.
+        private const string RequiredCompiledString = "Solidity files successfully";
+        // When script is done, it prints the ready-string.
+        private const string ReadyString = "Done! Sleeping indefinitely...";
+        private readonly Action<string> debug;
 
-        public ContractsReadyLogHandler(string targetString)
+        public ContractsReadyLogHandler(Action<string> debug)
         {
-            this.targetString = targetString;
+            this.debug = debug;
+            debug($"Looking for '{RequiredCompiledString}' and '{ReadyString}' in container logs...");
         }
 
+        public bool SeenCompileString { get; private set; }
         public bool Found { get; private set; }
 
         protected override void ProcessLine(string line)
         {
-            if (line.Contains(targetString)) Found = true;
+            debug(line);
+            if (line.Contains(RequiredCompiledString)) SeenCompileString = true;
+
+            if (SeenCompileString && line.Contains(ReadyString)) Found = true;
         }
     }
 }

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -56,30 +56,6 @@ namespace DistTestCore.Marketplace
             return marketplaceAbi;
         }
 
-        private string Retry(Func<string> fetch)
-        {
-            var result = string.Empty;
-            Time.WaitUntil(() =>
-            {
-                result = Catch(fetch);
-                return !string.IsNullOrEmpty(result);
-            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
-
-            return result;
-        }
-
-        private string Catch(Func<string> fetch)
-        {
-            try
-            {
-                return fetch();
-            }
-            catch
-            {
-                return string.Empty;
-            }
-        }
-
         private string FetchAccountsCsv()
         {
             return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename);
@@ -103,7 +79,7 @@ namespace DistTestCore.Marketplace
 
         private string FetchPubKey()
         {
-            var enodeFinder = new PubKeyFinder();
+            var enodeFinder = new PubKeyFinder(s => log.Debug(s));
             workflow.DownloadContainerLog(container, enodeFinder);
             return enodeFinder.GetPubKey();
         }
@@ -116,21 +92,35 @@ namespace DistTestCore.Marketplace
             var privateKey = tokens[1];
             return new GethAccount(account, privateKey);
         }
+
+        private static string Retry(Func<string> fetch)
+        {
+            return Time.Retry(fetch, nameof(ContainerInfoExtractor));
+        }
     }
 
     public class PubKeyFinder : LogHandler, ILogHandler
     {
         private const string openTag = "self=enode://";
         private const string openTagQuote = "self=\"enode://";
+        private readonly Action<string> debug;
         private string pubKey = string.Empty;
 
+        public PubKeyFinder(Action<string> debug)
+        {
+            this.debug = debug;
+            debug($"Looking for '{openTag}' in container logs...");
+        }
+
         public string GetPubKey()
         {
+            if (string.IsNullOrEmpty(pubKey)) throw new Exception("Not found yet exception.");
             return pubKey;
         }
 
         protected override void ProcessLine(string line)
         {
+            debug(line);
             if (line.Contains(openTag))
             {
                 ExtractPubKey(openTag, line);

DistTestCore/Marketplace/GethBootstrapNodeInfo.cs

@@ -1,5 +1,4 @@
 using KubernetesWorkflow;
-using Logging;
 using NethereumWorkflow;
 
 namespace DistTestCore.Marketplace
@@ -21,13 +20,12 @@ namespace DistTestCore.Marketplace
         public string PubKey { get; }
         public Port DiscoveryPort { get; }
 
-        public NethereumInteraction StartInteraction(BaseLog log)
+        public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
         {
-            var ip = RunningContainers.RunningPod.Cluster.IP;
+            var address = lifecycle.Configuration.GetAddress(RunningContainers.Containers[0]);
-            var port = RunningContainers.Containers[0].ServicePorts[0].Number;
             var account = Account;
 
-            var creator = new NethereumInteractionCreator(log, ip, port, account.PrivateKey);
+            var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, account.PrivateKey);
             return creator.CreateWorkflow();
         }
     }

DistTestCore/Marketplace/GethCompanionNodeInfo.cs

@@ -1,5 +1,4 @@
 using KubernetesWorkflow;
-using Logging;
 using NethereumWorkflow;
 
 namespace DistTestCore.Marketplace
@@ -15,13 +14,12 @@ namespace DistTestCore.Marketplace
         public RunningContainer RunningContainer { get; }
         public GethAccount[] Accounts { get; }
         
-        public NethereumInteraction StartInteraction(BaseLog log, GethAccount account)
+        public NethereumInteraction StartInteraction(TestLifecycle lifecycle, GethAccount account)
         {
-            var ip = RunningContainer.Pod.Cluster.IP;
+            var address = lifecycle.Configuration.GetAddress(RunningContainer);
-            var port = RunningContainer.ServicePorts[0].Number;
             var privateKey = account.PrivateKey;
 
-            var creator = new NethereumInteractionCreator(log, ip, port, privateKey);
+            var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, privateKey);
             return creator.CreateWorkflow();
         }
     }

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace
             {
                 Time.WaitUntil(() =>
                 {
-                    var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
+                    var interaction = node.StartInteraction(lifecycle, node.Accounts.First());
                     return interaction.IsSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
                 }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
             }

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -58,7 +58,7 @@ namespace DistTestCore.Marketplace
             var httpPort = AddExposedPort(tag: HttpPortTag);
 
             var bootPubKey = config.BootstrapNode.PubKey;
-            var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip;
+            var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.PodInfo.Ip;
             var bootPort = config.BootstrapNode.DiscoveryPort.Number;
             var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}";
 

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -1,5 +1,5 @@
 using DistTestCore.Codex;
-using Logging;
+using DistTestCore.Helpers;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
 using System.Numerics;
@@ -17,14 +17,14 @@ namespace DistTestCore.Marketplace
 
     public class MarketplaceAccess : IMarketplaceAccess
     {
-        private readonly TestLog log;
+        private readonly TestLifecycle lifecycle;
         private readonly MarketplaceNetwork marketplaceNetwork;
         private readonly GethAccount account;
         private readonly CodexAccess codexAccess;
 
-        public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
+        public MarketplaceAccess(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
         {
-            this.log = log;
+            this.lifecycle = lifecycle;
             this.marketplaceNetwork = marketplaceNetwork;
             this.account = account;
             this.codexAccess = codexAccess;
@@ -98,12 +98,12 @@ namespace DistTestCore.Marketplace
 
         public void AssertThatBalance(IResolveConstraint constraint, string message = "")
         {
-            Assert.That(GetBalance(), constraint, message);
+            AssertHelpers.RetryAssert(constraint, GetBalance, message);
         }
 
         public TestToken GetBalance()
         {
-            var interaction = marketplaceNetwork.StartInteraction(log);
+            var interaction = marketplaceNetwork.StartInteraction(lifecycle);
             var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account.Account);
             var balance = new TestToken(amount);
 
@@ -114,7 +114,7 @@ namespace DistTestCore.Marketplace
 
         private void Log(string msg)
         {
-            log.Log($"{codexAccess.Container.Name} {msg}");
+            lifecycle.Log.Log($"{codexAccess.Container.Name} {msg}");
         }
     }
 

DistTestCore/Marketplace/MarketplaceAccessFactory.cs

@@ -18,19 +18,19 @@ namespace DistTestCore.Marketplace
 
     public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory
     {
-        private readonly TestLog log;
+        private readonly TestLifecycle lifecycle;
         private readonly MarketplaceNetwork marketplaceNetwork;
 
-        public GethMarketplaceAccessFactory(TestLog log, MarketplaceNetwork marketplaceNetwork)
+        public GethMarketplaceAccessFactory(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork)
         {
-            this.log = log;
+            this.lifecycle = lifecycle;
             this.marketplaceNetwork = marketplaceNetwork;
         }
 
         public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
         {
             var companionNode = GetGethCompanionNode(access);
-            return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access);
+            return new MarketplaceAccess(lifecycle, marketplaceNetwork, companionNode, access);
         }
 
         private GethAccount GetGethCompanionNode(CodexAccess access)

DistTestCore/Marketplace/MarketplaceNetwork.cs

@@ -1,5 +1,4 @@
-using Logging;
+using NethereumWorkflow;
-using NethereumWorkflow;
 
 namespace DistTestCore.Marketplace
 {
@@ -14,9 +13,9 @@ namespace DistTestCore.Marketplace
         public GethBootstrapNodeInfo Bootstrap { get; }
         public MarketplaceInfo Marketplace { get; }
 
-        public NethereumInteraction StartInteraction(BaseLog log)
+        public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
         {
-            return Bootstrap.StartInteraction(log);
+            return Bootstrap.StartInteraction(lifecycle);
         }
     }
 }

DistTestCore/Metrics/MetricsAccess.cs

@@ -1,4 +1,5 @@
-using KubernetesWorkflow;
+using DistTestCore.Helpers;
+using KubernetesWorkflow;
 using Logging;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
@@ -28,12 +29,14 @@ namespace DistTestCore.Metrics
 
         public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
         {
-            var metricSet = GetMetricWithTimeout(metricName);
+            AssertHelpers.RetryAssert(constraint, () =>
-            var metricValue = metricSet.Values[0].Value;
+            {
+                var metricSet = GetMetricWithTimeout(metricName);
+                var metricValue = metricSet.Values[0].Value;
 
-            log.Log($"{node.Name} metric '{metricName}' = {metricValue}");
+                log.Log($"{node.Name} metric '{metricName}' = {metricValue}");
-
+                return metricValue;
-            Assert.That(metricValue, constraint, message);
+            }, message);
         }
 
         public Metrics? GetAllMetrics()

DistTestCore/Metrics/MetricsAccessFactory.cs

@@ -28,7 +28,7 @@ namespace DistTestCore.Metrics
 
         public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
         {
-            var query = new MetricsQuery(lifecycle.Log, lifecycle.TimeSet, prometheusContainer);
+            var query = new MetricsQuery(lifecycle, prometheusContainer);
             return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
         }
     }

DistTestCore/Metrics/MetricsQuery.cs

@@ -1,6 +1,5 @@
 using DistTestCore.Codex;
 using KubernetesWorkflow;
-using Logging;
 using System.Globalization;
 
 namespace DistTestCore.Metrics
@@ -9,15 +8,16 @@ namespace DistTestCore.Metrics
     {
         private readonly Http http;
 
-        public MetricsQuery(BaseLog log, ITimeSet timeSet, RunningContainers runningContainers)
+        public MetricsQuery(TestLifecycle lifecycle, RunningContainers runningContainers)
         {
             RunningContainers = runningContainers;
 
+            var address = lifecycle.Configuration.GetAddress(runningContainers.Containers[0]);
+
             http = new Http(
-                log,
+                lifecycle.Log,
-                timeSet,
+                lifecycle.TimeSet,
-                runningContainers.RunningPod.Cluster.IP,
+                address,
-                runningContainers.Containers[0].ServicePorts[0].Number,
                 "api/v1");
         }
 
@@ -119,7 +119,7 @@ namespace DistTestCore.Metrics
 
         private string GetInstanceNameForNode(RunningContainer node)
         {
-            var ip = node.Pod.Ip;
+            var ip = node.Pod.PodInfo.Ip;
             var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
             return $"{ip}:{port}";
         }

DistTestCore/OnlineCodexNode.cs

@@ -10,6 +10,8 @@ namespace DistTestCore
     {
         string GetName();
         CodexDebugResponse GetDebugInfo();
+        CodexDebugPeerResponse GetDebugPeer(string peerId);
+        CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout);
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId);
         void ConnectToPeer(IOnlineCodexNode node);
@@ -47,10 +49,21 @@ namespace DistTestCore
         public CodexDebugResponse GetDebugInfo()
         {
             var debugInfo = CodexAccess.GetDebugInfo();
-            Log($"Got DebugInfo with id: '{debugInfo.id}'.");
+            var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
+            Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
             return debugInfo;
         }
 
+        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        {
+            return CodexAccess.GetDebugPeer(peerId);
+        }
+
+        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
+        {
+            return CodexAccess.GetDebugPeer(peerId, timeout);
+        }
+
         public ContentId UploadFile(TestFile file)
         {
             Log($"Uploading file of size {file.GetFileSize()}...");
@@ -111,7 +124,7 @@ namespace DistTestCore
 
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.Ip);
+            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip);
         }
 
         private void DownloadToFile(string contentId, TestFile file)

DistTestCore/PrometheusStarter.cs

@@ -44,7 +44,7 @@ namespace DistTestCore
 
             foreach (var node in nodes)
             {
-                var ip = node.Pod.Ip;
+                var ip = node.Pod.PodInfo.Ip;
                 var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
                 config += $"          - '{ip}:{port}'\n";
             }

DistTestCore/TestLifecycle.cs

@@ -13,6 +13,7 @@ namespace DistTestCore
         public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
         {
             Log = log;
+            Configuration = configuration;
             TimeSet = timeSet;
             workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet));
 
@@ -24,6 +25,7 @@ namespace DistTestCore
         }
 
         public TestLog Log { get; }
+        public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
         public FileManager FileManager { get; }
         public CodexStarter CodexStarter { get; }

DistTestCore/Timing.cs

@@ -1,5 +1,4 @@
 using NUnit.Framework;
-using Utils;
 
 namespace DistTestCore
 {
@@ -11,8 +10,8 @@ namespace DistTestCore
     public interface ITimeSet
     {
         TimeSpan HttpCallTimeout();
-        int HttpCallRetryCount();
+        TimeSpan HttpCallRetryTimeout();
-        void HttpCallRetryDelay();
+        TimeSpan HttpCallRetryDelay();
         TimeSpan WaitForK8sServiceDelay();
         TimeSpan K8sOperationTimeout();
         TimeSpan WaitForMetricTimeout();
@@ -25,14 +24,14 @@ namespace DistTestCore
             return TimeSpan.FromSeconds(10);
         }
 
-        public int HttpCallRetryCount()
+        public TimeSpan HttpCallRetryTimeout()
         {
-            return 5;
+            return TimeSpan.FromMinutes(1);
         }
 
-        public void HttpCallRetryDelay()
+        public TimeSpan HttpCallRetryDelay()
         {
-            Time.Sleep(TimeSpan.FromSeconds(3));
+            return TimeSpan.FromSeconds(3);
         }
 
         public TimeSpan WaitForK8sServiceDelay()
@@ -58,14 +57,14 @@ namespace DistTestCore
             return TimeSpan.FromHours(2);
         }
 
-        public int HttpCallRetryCount()
+        public TimeSpan HttpCallRetryTimeout()
         {
-            return 2;
+            return TimeSpan.FromHours(5);
         }
 
-        public void HttpCallRetryDelay()
+        public TimeSpan HttpCallRetryDelay()
         {
-            Time.Sleep(TimeSpan.FromMinutes(5));
+            return TimeSpan.FromMinutes(5);
         }
 
         public TimeSpan WaitForK8sServiceDelay()

KubernetesWorkflow/ApplicationLifecycle.cs

@@ -1,40 +0,0 @@
-using Utils;
-
-namespace KubernetesWorkflow
-{
-    public class ApplicationLifecycle
-    {
-        private static object instanceLock = new object();
-        private static ApplicationLifecycle? instance;
-        private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
-        private readonly NumberSource namespaceNumberSource = new NumberSource(0);
-
-        private ApplicationLifecycle()
-        {
-        }
-
-        public static ApplicationLifecycle Instance
-        {
-            // I know singletons are quite evil. But we need to be sure this object is created only once
-            // and persists for the entire application lifecycle.
-            get
-            {
-                lock (instanceLock)
-                {
-                    if (instance == null) instance = new ApplicationLifecycle();
-                    return instance;
-                }
-            }
-        }
-
-        public NumberSource GetServiceNumberSource()
-        {
-            return servicePortNumberSource;
-        }
-
-        public string GetTestNamespace()
-        {
-            return namespaceNumberSource.GetNextNumber().ToString("D5");
-        }
-    }
-}

KubernetesWorkflow/CommandRunner.cs

@@ -28,7 +28,7 @@ namespace KubernetesWorkflow
             var input = new[] { command }.Concat(arguments).ToArray();
 
             Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
-                pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
+                pod.PodInfo.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
         }
 
         public string GetStdOut()

KubernetesWorkflow/Configuration.cs

@@ -2,31 +2,17 @@
 {
     public class Configuration
     {
-        public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
+        public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay)
         {
             K8sNamespacePrefix = k8sNamespacePrefix;
             KubeConfigFile = kubeConfigFile;
             OperationTimeout = operationTimeout;
             RetryDelay = retryDelay;
-            LocationMap = locationMap;
         }
 
         public string K8sNamespacePrefix { get; }
         public string? KubeConfigFile { get; }
         public TimeSpan OperationTimeout { get; }
         public TimeSpan RetryDelay { get; }
-        public ConfigurationLocationEntry[] LocationMap { get; }
-    }
-
-    public class ConfigurationLocationEntry
-    {
-        public ConfigurationLocationEntry(Location location, string workerName)
-        {
-            Location = location;
-            WorkerName = workerName;
-        }
-
-        public Location Location { get; }
-        public string WorkerName { get; }
     }
 }

KubernetesWorkflow/K8sCluster.cs

@@ -10,19 +10,28 @@ namespace KubernetesWorkflow
         }
        
         public Configuration Configuration { get; }
-        public string IP { get; private set; } = string.Empty;
+        public string HostAddress { get; private set; } = string.Empty;
+        public K8sNodeLabel[] AvailableK8sNodes { get; set; } = new K8sNodeLabel[0];
 
         public KubernetesClientConfiguration GetK8sClientConfig()
         {
             var config = GetConfig();
-            UpdateIp(config);
+            UpdateHostAddress(config);
             return config;
         }
 
-        public string GetNodeLabelForLocation(Location location)
+        public K8sNodeLabel? GetNodeLabelForLocation(Location location)
         {
-            if (location == Location.Unspecified) return string.Empty;
+            switch (location)
-            return Configuration.LocationMap.Single(l => l.Location == location).WorkerName;
+            {
+                case Location.One:
+                    return K8sNodeIfAvailable(0);
+                case Location.Two:
+                    return K8sNodeIfAvailable(1);
+                case Location.Three:
+                    return K8sNodeIfAvailable(2);
+            }
+            return null;
         }
 
         public TimeSpan K8sOperationTimeout()
@@ -47,10 +56,35 @@ namespace KubernetesWorkflow
             }
         }
 
-        private void UpdateIp(KubernetesClientConfiguration config)
+        private void UpdateHostAddress(KubernetesClientConfiguration config)
         {
             var host = config.Host.Replace("https://", "");
-            IP = host.Substring(0, host.IndexOf(':'));
+            if (host.Contains(":"))
+            {
+                HostAddress = "http://" + host.Substring(0, host.IndexOf(':'));
+            }
+            else
+            {
+                HostAddress = config.Host;
+            }
+        }
+
+        private K8sNodeLabel? K8sNodeIfAvailable(int index)
+        {
+            if (AvailableK8sNodes.Length <= index) return null;
+            return AvailableK8sNodes[index];
         }
     }
+
+    public class K8sNodeLabel
+    {
+        public K8sNodeLabel(string key, string value)
+        {
+            Key = key;
+            Value = value;
+        }
+
+        public string Key { get; }
+        public string Value { get; }
+    }
 }

KubernetesWorkflow/K8sController.cs

@@ -22,7 +22,6 @@ namespace KubernetesWorkflow
             client = new K8sClient(cluster.GetK8sClientConfig());
 
             K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
-            log.Debug($"Test namespace: '{K8sTestNamespace}'");
         }
 
         public void Dispose()
@@ -33,13 +32,14 @@ namespace KubernetesWorkflow
         public RunningPod BringOnline(ContainerRecipe[] containerRecipes, Location location)
         {
             log.Debug();
+            DiscoverK8sNodes();
             EnsureTestNamespace();
 
             var deploymentName = CreateDeployment(containerRecipes, location);
             var (serviceName, servicePortsMap) = CreateService(containerRecipes);
-            var (podName, podIp) = FetchNewPod();
+            var podInfo = FetchNewPod();
 
-            return new RunningPod(cluster, podName, podIp, deploymentName, serviceName, servicePortsMap);
+            return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap);
         }
 
         public void Stop(RunningPod pod)
@@ -48,22 +48,27 @@ namespace KubernetesWorkflow
             if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName);
             DeleteDeployment(pod.DeploymentName);
             WaitUntilDeploymentOffline(pod.DeploymentName);
-            WaitUntilPodOffline(pod.Name);
+            WaitUntilPodOffline(pod.PodInfo.Name);
         }
 
         public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
         {
             log.Debug();
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name));
             logHandler.Log(stream);
         }
 
         public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
         {
-            log.Debug($"{containerName}: {command} ({string.Join(",", args)})");
+            var cmdAndArgs = $"{containerName}: {command} ({string.Join(",", args)})";
+            log.Debug(cmdAndArgs);
+
             var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args);
             runner.Run();
-            return runner.GetStdOut();
+            var result = runner.GetStdOut();
+
+            log.Debug($"{cmdAndArgs} = '{result}'");
+            return result;
         }
 
         public void DeleteAllResources()
@@ -102,6 +107,42 @@ namespace KubernetesWorkflow
             }
         }
 
+        #region Discover K8s Nodes
+
+        private void DiscoverK8sNodes()
+        {
+            if (cluster.AvailableK8sNodes == null || !cluster.AvailableK8sNodes.Any())
+            {
+                cluster.AvailableK8sNodes = GetAvailableK8sNodes();
+                if (cluster.AvailableK8sNodes.Length < 3)
+                {
+                    log.Debug($"Warning: For full location support, at least 3 Kubernetes Nodes are required in the cluster. Nodes found: '{string.Join(",", cluster.AvailableK8sNodes.Select(p => $"{p.Key}={p.Value}"))}'.");
+                }
+            }
+        }
+
+        private K8sNodeLabel[] GetAvailableK8sNodes()
+        {
+            var nodes = client.Run(c => c.ListNode());
+
+            var optionals = nodes.Items.Select(i => CreateNodeLabel(i));
+            return optionals.Where(n => n != null).Select(n => n!).ToArray();
+        }
+
+        private K8sNodeLabel? CreateNodeLabel(V1Node i)
+        {
+            var keys = i.Metadata.Labels.Keys;
+            var hostnameKey = keys.SingleOrDefault(k => k.ToLowerInvariant().Contains("hostname"));
+            if (hostnameKey != null)
+            {
+                var hostnameValue = i.Metadata.Labels[hostnameKey];
+                return new K8sNodeLabel(hostnameKey, hostnameValue);
+            }
+            return null;
+        }
+
+        #endregion
+
         #region Namespace management
 
         private string K8sTestNamespace { get; }
@@ -148,10 +189,7 @@ namespace KubernetesWorkflow
                     },
                     Spec = new V1NetworkPolicySpec
                     {
-                        PodSelector = new V1LabelSelector
+                        PodSelector = new V1LabelSelector {},
-                        {
-                            MatchLabels = GetSelector()
-                        },
                         PolicyTypes = new[]
                         {
                             "Ingress",
@@ -159,6 +197,16 @@ namespace KubernetesWorkflow
                         },
                         Ingress = new List<V1NetworkPolicyIngressRule>
                         {
+                            new V1NetworkPolicyIngressRule
+                            {
+                                FromProperty = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        PodSelector = new V1LabelSelector {}
+                                    }
+                                }
+                            },
                             new V1NetworkPolicyIngressRule
                             {
                                 FromProperty = new List<V1NetworkPolicyPeer>
@@ -167,7 +215,7 @@ namespace KubernetesWorkflow
                                     {
                                         NamespaceSelector = new V1LabelSelector
                                         {
-                                            MatchLabels = GetMyNamespaceSelector()
+                                            MatchLabels = GetRunnerNamespaceSelector()
                                         }
                                     }
                                 }
@@ -175,6 +223,16 @@ namespace KubernetesWorkflow
                         },
                         Egress = new List<V1NetworkPolicyEgressRule>
                         {
+                            new V1NetworkPolicyEgressRule
+                            {
+                                To = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        PodSelector = new V1LabelSelector {}
+                                    }
+                                }
+                            },
                             new V1NetworkPolicyEgressRule
                             {
                                 To = new List<V1NetworkPolicyPeer>
@@ -183,11 +241,62 @@ namespace KubernetesWorkflow
                                     {
                                         NamespaceSelector = new V1LabelSelector
                                         {
-                                            MatchLabels = GetMyNamespaceSelector()
+                                            MatchLabels = new Dictionary<string, string> { { "kubernetes.io/metadata.name", "kube-system" } }
                                         }
+                                    },
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        PodSelector = new V1LabelSelector
+                                        {
+                                            MatchLabels = new Dictionary<string, string> { { "k8s-app", "kube-dns" } }
+                                        }
+                                    }
+                                },
+                                Ports = new List<V1NetworkPolicyPort>
+                                {
+                                    new V1NetworkPolicyPort
+                                    {
+                                        Port = new IntstrIntOrString
+                                        {
+                                            Value = "53"
+                                        },
+                                        Protocol = "UDP"
+                                    }
+                                }
+                            },
+                            new V1NetworkPolicyEgressRule
+                            {
+                                To = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        IpBlock = new V1IPBlock
+                                        {
+                                          Cidr = "0.0.0.0/0"
+                                        }
+                                    }
+                                },
+                                Ports = new List<V1NetworkPolicyPort>
+                                {
+                                    new V1NetworkPolicyPort
+                                    {
+                                        Port = new IntstrIntOrString
+                                        {
+                                            Value = "80"
+                                        },
+                                        Protocol = "TCP"
+                                    },
+                                    new V1NetworkPolicyPort
+                                    {
+                                        Port = new IntstrIntOrString
+                                        {
+                                            Value = "443"
+                                        },
+                                        Protocol = "TCP"
                                     }
                                 }
                             }
+
                         }
                     }
                 };
@@ -242,11 +351,12 @@ namespace KubernetesWorkflow
 
         private IDictionary<string, string> CreateNodeSelector(Location location)
         {
-            if (location == Location.Unspecified) return new Dictionary<string, string>();
+            var nodeLabel = cluster.GetNodeLabelForLocation(location);
+            if (nodeLabel == null) return new Dictionary<string, string>();
 
             return new Dictionary<string, string>
             {
-                { "codex-test-location", cluster.GetNodeLabelForLocation(location) }
+                { nodeLabel.Key, nodeLabel.Value }
             };
         }
 
@@ -255,9 +365,9 @@ namespace KubernetesWorkflow
             return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
         }
 
-        private IDictionary<string, string> GetMyNamespaceSelector()
+        private IDictionary<string, string> GetRunnerNamespaceSelector()
         {
-            return new Dictionary<string, string> { { "name", "thatisincorrect" } };
+            return new Dictionary<string, string> { { "kubernetes.io/metadata.name", "default" } };
         }
 
         private V1ObjectMeta CreateDeploymentMetadata()
@@ -329,11 +439,11 @@ namespace KubernetesWorkflow
         {
             var result = new Dictionary<ContainerRecipe, Port[]>();
 
-            var ports = CreateServicePorts(result, containerRecipes);
+            var ports = CreateServicePorts(containerRecipes);
 
             if (!ports.Any())
             {
-                // None of these container-recipes wish to expose anything via a serice port.
+                // None of these container-recipes wish to expose anything via a service port.
                 // So, we don't have to create a service.
                 return (string.Empty, result);
             }
@@ -352,9 +462,40 @@ namespace KubernetesWorkflow
 
             client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace));
 
+            ReadBackServiceAndMapPorts(serviceSpec, containerRecipes, result);
+
             return (serviceSpec.Metadata.Name, result);
         }
 
+        private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, Dictionary<ContainerRecipe, Port[]> result)
+        {
+            // For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
+            var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sTestNamespace));
+            foreach (var r in containerRecipes)
+            {
+                if (r.ExposedPorts.Any())
+                {
+                    var firstExposedPort = r.ExposedPorts.First();
+                    var portName = GetNameForPort(r, firstExposedPort);
+
+                    var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName);
+                    if (matchingServicePorts.Any())
+                    {
+                        // These service ports belongs to this recipe.
+                        var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
+                        var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
+                        result.Add(r, ports);
+                    }
+                }
+            }
+        }
+
+        private Port? MapNodePortIfAble(V1ServicePort p, string tag)
+        {
+            if (p.NodePort == null) return null;
+            return new Port(p.NodePort.Value, tag);
+        }
+
         private void DeleteService(string serviceName)
         {
             client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace));
@@ -369,36 +510,30 @@ namespace KubernetesWorkflow
             };
         }
 
-        private List<V1ServicePort> CreateServicePorts(Dictionary<ContainerRecipe, Port[]> servicePorts, ContainerRecipe[] recipes)
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes)
         {
             var result = new List<V1ServicePort>();
             foreach (var recipe in recipes)
             {
-                result.AddRange(CreateServicePorts(servicePorts, recipe));
+                result.AddRange(CreateServicePorts(recipe));
             }
             return result;
         }
 
-        private List<V1ServicePort> CreateServicePorts(Dictionary<ContainerRecipe, Port[]> servicePorts, ContainerRecipe recipe)
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe)
         {
             var result = new List<V1ServicePort>();
-            var usedPorts = new List<Port>();
             foreach (var port in recipe.ExposedPorts)
             {
-                var servicePort = workflowNumberSource.GetServicePort();
-                usedPorts.Add(new Port(servicePort, ""));
-
                 result.Add(new V1ServicePort
                 {
                     Name = GetNameForPort(recipe, port),
                     Protocol = "TCP",
                     Port = port.Number,
                     TargetPort = GetNameForPort(recipe, port),
-                    NodePort = servicePort
                 });                
             }
 
-            servicePorts.Add(recipe, usedPorts.ToArray());
             return result;
         }
 
@@ -465,7 +600,7 @@ namespace KubernetesWorkflow
 
         #endregion
 
-        private (string, string) FetchNewPod()
+        private PodInfo FetchNewPod()
         {
             var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
 
@@ -475,12 +610,13 @@ namespace KubernetesWorkflow
             var newPod = newPods.Single();
             var name = newPod.Name();
             var ip = newPod.Status.PodIP;
+            var k8sNodeName = newPod.Spec.NodeName;
 
             if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure.");
             if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure.");
 
             knownPods.Add(name);
-            return (name, ip);
+            return new PodInfo(name, ip, k8sNodeName);
         }
     }
 }

KubernetesWorkflow/Location.cs

@@ -3,7 +3,8 @@
     public enum Location
     {
         Unspecified,
-        BensLaptop,
+        One,
-        BensOldGamingMachine
+        Two,
+        Three,
     }
 }

KubernetesWorkflow/RunningContainers.cs

@@ -21,18 +21,22 @@
 
     public class RunningContainer
     {
-        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig)
+        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig, RunningContainerAddress clusterExternalAddress, RunningContainerAddress clusterInternalAddress)
         {
             Pod = pod;
             Recipe = recipe;
             ServicePorts = servicePorts;
             Name = GetContainerName(recipe, startupConfig);
+            ClusterExternalAddress = clusterExternalAddress;
+            ClusterInternalAddress = clusterInternalAddress;
         }
 
         public string Name { get; }
         public RunningPod Pod { get; }
         public ContainerRecipe Recipe { get; }
         public Port[] ServicePorts { get; }
+        public RunningContainerAddress ClusterExternalAddress { get; }
+        public RunningContainerAddress ClusterInternalAddress { get; }
 
         private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
         {
@@ -46,4 +50,16 @@
             }
         }
     }
+
+    public class RunningContainerAddress
+    {
+        public RunningContainerAddress(string host, int port)
+        {
+            Host = host;
+            Port = port;
+        }
+
+        public string Host { get; }
+        public int Port { get; }
+    }
 }

KubernetesWorkflow/RunningPod.cs

@@ -4,25 +4,38 @@
     {
         private readonly Dictionary<ContainerRecipe, Port[]> servicePortMap;
 
-        public RunningPod(K8sCluster cluster, string name, string ip, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
+        public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName, Dictionary<ContainerRecipe, Port[]> servicePortMap)
         {
             Cluster = cluster;
-            Name = name;
+            PodInfo = podInfo;
-            Ip = ip;
             DeploymentName = deploymentName;
             ServiceName = serviceName;
             this.servicePortMap = servicePortMap;
         }
 
         public K8sCluster Cluster { get; }
-        public string Name { get; }
+        public PodInfo PodInfo { get; }
-        public string Ip { get; }
         internal string DeploymentName { get; }
         internal string ServiceName { get; }
 
         public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
         {
+            if (!servicePortMap.ContainsKey(containerRecipe)) return Array.Empty<Port>();
             return servicePortMap[containerRecipe];
         }
     }
+
+    public class PodInfo
+    {
+        public PodInfo(string podName, string podIp, string k8sNodeName)
+        {
+            Name = podName;
+            Ip = podIp;
+            K8SNodeName = k8sNodeName;
+        }
+
+        public string Name { get; }
+        public string Ip { get; }
+        public string K8SNodeName { get; }
+    }
 }

KubernetesWorkflow/StartupWorkflow.cs

@@ -80,10 +80,43 @@ namespace KubernetesWorkflow
                 var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
                 log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
 
-                return new RunningContainer(runningPod, r, servicePorts, startupConfig);
+                return new RunningContainer(runningPod, r, servicePorts, startupConfig,
+                    GetContainerExternalAddress(runningPod, servicePorts),
+                    GetContainerInternalAddress(r));
+
             }).ToArray();
         }
 
+        private RunningContainerAddress GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
+        {
+            return new RunningContainerAddress(
+                pod.Cluster.HostAddress,
+                GetServicePort(servicePorts));
+        }
+
+        private RunningContainerAddress GetContainerInternalAddress(ContainerRecipe recipe)
+        {
+            var serviceName = "service-" + numberSource.WorkflowNumber;
+            var namespaceName = cluster.Configuration.K8sNamespacePrefix + testNamespace;
+            var port = GetInternalPort(recipe);
+
+            return new RunningContainerAddress(
+                $"http://{serviceName}.{namespaceName}.svc.cluster.local",
+                port);
+        }
+
+        private static int GetServicePort(Port[] servicePorts)
+        {
+            if (servicePorts.Any()) return servicePorts.First().Number;
+            return 0;
+        }
+
+        private static int GetInternalPort(ContainerRecipe recipe)
+        {
+            if (recipe.ExposedPorts.Any()) return recipe.ExposedPorts.First().Number;
+            return 0;
+        }
+
         private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             log.Debug();

KubernetesWorkflow/WorkflowCreator.cs

@@ -16,13 +16,12 @@ namespace KubernetesWorkflow
         {
             cluster = new K8sCluster(configuration);
             this.log = log;
-            testNamespace = ApplicationLifecycle.Instance.GetTestNamespace();
+            testNamespace = Guid.NewGuid().ToString().ToLowerInvariant();
         }
 
         public StartupWorkflow CreateWorkflow()
         {
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
-                                                                    ApplicationLifecycle.Instance.GetServiceNumberSource(),
                                                                     containerNumberSource);
 
             return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);

KubernetesWorkflow/WorkflowNumberSource.cs

@@ -4,13 +4,11 @@ namespace KubernetesWorkflow
 {
     public class WorkflowNumberSource
     {
-        private readonly NumberSource servicePortNumberSource;
         private readonly NumberSource containerNumberSource;
 
-        public WorkflowNumberSource(int workflowNumber, NumberSource servicePortNumberSource, NumberSource containerNumberSource)
+        public WorkflowNumberSource(int workflowNumber, NumberSource containerNumberSource)
         {
             WorkflowNumber = workflowNumber;
-            this.servicePortNumberSource = servicePortNumberSource;
             this.containerNumberSource = containerNumberSource;
         }
 
@@ -20,10 +18,5 @@ namespace KubernetesWorkflow
         {
             return containerNumberSource.GetNextNumber();
         }
-
-        public int GetServicePort()
-        {
-            return servicePortNumberSource.GetNextNumber();
-        }
     }
 }

Logging/BaseLog.cs

@@ -54,6 +54,7 @@ namespace Logging
 
         public void AddStringReplace(string from, string to)
         {
+            if (string.IsNullOrWhiteSpace(from)) return;
             replacements.Add(new BaseLogStringReplacement(from, to));
         }
 

LongTests/BasicTests/DownloadTests.cs

@@ -1,7 +1,7 @@
 using DistTestCore;
 using NUnit.Framework;
 
-namespace Tests.ParallelTests
+namespace TestsLong.BasicTests
 {
     [TestFixture]
     public class DownloadTests : DistTest
@@ -23,7 +23,7 @@ namespace Tests.ParallelTests
             var testFile = GenerateTestFile(filesizeMb.MB());
             var contentId = host.UploadFile(testFile);
             var list = new List<Task<TestFile?>>();
-            
+
             foreach (var node in group)
             {
                 list.Add(Task.Run(() => { return node.DownloadContent(contentId); }));

LongTests/BasicTests/LargeFileTests.cs

@@ -1,5 +1,4 @@
 using DistTestCore;
-using DistTestCore.Codex;
 using NUnit.Framework;
 
 namespace TestsLong.BasicTests
@@ -11,7 +10,6 @@ namespace TestsLong.BasicTests
         public void OneClientLargeFileTest()
         {
             var primary = SetupCodexNode(s => s
-                                .WithLogLevel(CodexLogLevel.Warn)
                                 .WithStorageQuota(20.GB()));
 
             var testFile = GenerateTestFile(10.GB());

LongTests/BasicTests/TestInfraTests.cs

@@ -1,5 +1,4 @@
 using DistTestCore;
-using DistTestCore.Codex;
 using NUnit.Framework;
 
 namespace TestsLong.BasicTests
@@ -32,7 +31,6 @@ namespace TestsLong.BasicTests
         public void DownloadConsistencyTest()
         {
             var primary = SetupCodexNode(s => s
-                            .WithLogLevel(CodexLogLevel.Trace)
                             .WithStorageQuota(2.MB()));
 
             var testFile = GenerateTestFile(1.MB());

LongTests/BasicTests/UploadTests.cs

@@ -1,7 +1,7 @@
 using DistTestCore;
 using NUnit.Framework;
 
-namespace Tests.ParallelTests
+namespace TestsLong.BasicTests
 {
     [TestFixture]
     public class UploadTests : DistTest

Nethereum/NethereumInteraction.cs

@@ -131,7 +131,7 @@ namespace NethereumWorkflow
     public class MintTokensFunction : FunctionMessage
     {
         [Parameter("address", "holder", 1)]
-        public string Holder { get; set; }
+        public string Holder { get; set; } = string.Empty;
 
         [Parameter("uint256", "amount", 2)]
         public BigInteger Amount { get; set; }
@@ -141,6 +141,6 @@ namespace NethereumWorkflow
     public class GetTokenBalanceFunction : FunctionMessage
     {
         [Parameter("address", "owner", 1)]
-        public string Owner { get; set; }
+        public string Owner { get; set; } = string.Empty;
     }
 }

Nethereum/NethereumInteractionCreator.cs

@@ -26,7 +26,7 @@ namespace NethereumWorkflow
         private Web3 CreateWeb3()
         {
             var account = new Nethereum.Web3.Accounts.Account(privateKey);
-            return new Web3(account, $"http://{ip}:{port}");
+            return new Web3(account, $"{ip}:{port}");
         }
     }
 }

README.md

@@ -1,11 +1,13 @@
 # Distributed System Tests for Nim-Codex
 
-Using a common dotnet unit-test framework and a few other libraries, this project allows you to write tests that use multiple Codex node instances in various configurations to test the distributed system in a controlled, reproducable environment.
 
-Nim-Codex: https://github.com/status-im/nim-codex
+Using a common dotnet unit-test framework and a few other libraries, this project allows you to write tests that use multiple Codex node instances in various configurations to test the distributed system in a controlled, reproducible environment.
-Dotnet: v6.0
+
-Kubernetes: v1.25.4
+
-Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp
+Nim-Codex: https://github.com/codex-storage/nim-codex  
+Dotnet: v6.0  
+Kubernetes: v1.25.4  
+Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp  
 Nethereum: v4.14.0
 
 ## Tests
@@ -15,6 +17,17 @@ Tests are devided into two assemblies: `/Tests` and `/LongTests`.
 
 TODO: All tests will eventually be running as part of a dedicated CI pipeline and kubernetes cluster. Currently, we're developing these tests and the infra-code to support it by running the whole thing locally.
 
+## Configuration
+Test executing can be configured using the following environment variables.
+| Variable       | Description                                                                                                                                          | Default             |
+|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|
+| KUBECONFIG     | Optional path (abs or rel) to kubeconfig YAML file. When null, uses system default (docker-desktop) kubeconfig if available.                         | (null)              |
+| LOGPATH        | Path (abs or rel) where log files will be saved.                                                                                                     | "CodexTestLogs"     |
+| LOGDEBUG       | When "true", enables additional test-runner debug log output.                                                                                                    | "false"             |
+| DATAFILEPATH   | Path (abs or rel) where temporary test data files will be saved.                                                                                     | "TestDataFiles"     |
+| LOGLEVEL       | Codex log-level. (case-insensitive)                                                                                                                  | "Trace"             |
+| RUNNERLOCATION | Use "ExternalToCluster" when test app is running outside of the k8s cluster. Use "InternalToCluster" when tests are run from inside a pod/container. | "ExternalToCluster" |
+
 ## Test logs
 Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs.
 

Tests/BasicTests/ExampleTests.cs

@@ -1,5 +1,4 @@
 using DistTestCore;
-using DistTestCore.Codex;
 using NUnit.Framework;
 using Utils;
 
@@ -11,7 +10,7 @@ namespace Tests.BasicTests
         [Test]
         public void CodexLogExample()
         {
-            var primary = SetupCodexNode(s => s.WithLogLevel(CodexLogLevel.Trace));
+            var primary = SetupCodexNode();
 
             primary.UploadFile(GenerateTestFile(5.MB()));
 
@@ -47,7 +46,6 @@ namespace Tests.BasicTests
             var buyerInitialBalance = 1000.TestTokens();
 
             var seller = SetupCodexNode(s => s
-                            .WithLogLevel(CodexLogLevel.Trace)
                             .WithStorageQuota(11.GB())
                             .EnableMarketplace(sellerInitialBalance));
 
@@ -61,7 +59,6 @@ namespace Tests.BasicTests
             var testFile = GenerateTestFile(10.MB());
 
             var buyer = SetupCodexNode(s => s
-                .WithLogLevel(CodexLogLevel.Trace)
                 .WithBootstrapNode(seller)
                 .EnableMarketplace(buyerInitialBalance));
             

Tests/BasicTests/PeerTests.cs

@@ -1,85 +0,0 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using NUnit.Framework;
-
-namespace Tests.BasicTests
-{
-    [TestFixture]
-    public class PeerTests : DistTest
-    {
-        [Test]
-        public void TwoNodes()
-        {
-            var primary = SetupCodexBootstrapNode();
-            var secondary = SetupCodexNode(s => s.WithBootstrapNode(primary));
-
-            primary.ConnectToPeer(secondary); // TODO REMOVE THIS: This is required for the switchPeers to show up.
-
-            // This is required for the enginePeers to show up.
-            //var file = GenerateTestFile(10.MB());
-            //var contentId = primary.UploadFile(file);
-            //var file2 = secondary.DownloadContent(contentId);
-            //file.AssertIsEqual(file2);
-
-            AssertKnowEachother(primary, secondary);
-        }
-
-        [TestCase(2)]
-        [TestCase(3)]
-        [TestCase(10)]
-        public void VariableNodes(int number)
-        {
-            var bootstrap = SetupCodexBootstrapNode();
-            var nodes = SetupCodexNodes(number, s => s.WithBootstrapNode(bootstrap));
-
-            var file = GenerateTestFile(10.MB());
-            var contentId = nodes.First().UploadFile(file);
-            var file2 = nodes.Last().DownloadContent(contentId);
-            file.AssertIsEqual(file2);
-
-            // <TODO REMOVE THIS>
-            foreach (var node in nodes) bootstrap.ConnectToPeer(node);
-            for (var x = 0; x < number; x++)
-            {
-                for (var y = x + 1; y < number; y++)
-                {
-                    nodes[x].ConnectToPeer(nodes[y]);
-                }
-            }
-            // </TODO REMOVE THIS>
-
-            foreach (var node in nodes) AssertKnowEachother(node, bootstrap);
-
-            for (var x = 0; x < number; x++)
-            {
-                for (var y = x + 1; y < number; y++)
-                {
-                    AssertKnowEachother(nodes[x], nodes[y]);
-                }
-            }
-        }
-
-        private void AssertKnowEachother(IOnlineCodexNode a, IOnlineCodexNode b)
-        {
-            AssertKnowEachother(a.GetDebugInfo(), b.GetDebugInfo());
-        }
-
-        private void AssertKnowEachother(CodexDebugResponse a, CodexDebugResponse b)
-        {
-            AssertKnows(a, b);
-            AssertKnows(b, a);
-        }
-
-        private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b)
-        {
-            var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
-            var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId));
-
-            Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]");
-            Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
-
-            Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
-            Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found.");
-        }
-    }
-}

Tests/BasicTests/TwoClientTests.cs

@@ -28,11 +28,10 @@ namespace Tests.BasicTests
         }
 
         [Test]
-        [Ignore("Requires Location map to be configured for k8s cluster.")]
         public void TwoClientsTwoLocationsTest()
         {
-            var primary = SetupCodexNode(s => s.At(Location.BensLaptop));
+            var primary = SetupCodexNode(s => s.At(Location.One));
-            var secondary = SetupCodexNode(s => s.At(Location.BensOldGamingMachine));
+            var secondary = SetupCodexNode(s => s.At(Location.Two));
 
             PerformTwoClientTest(primary, secondary);
         }

Tests/DurabilityTests/DurabilityTests.cs

@@ -1,66 +0,0 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using NUnit.Framework;
-using Utils;
-
-namespace Tests.DurabilityTests
-{
-    [TestFixture]
-    public class DurabilityTests : DistTest
-    {
-        [Test]
-        public void BootstrapNodeDisappearsTest()
-        {
-            var bootstrapNode = SetupCodexBootstrapNode();
-            var group = SetupCodexNodes(2, s => s.WithBootstrapNode(bootstrapNode));
-            var primary = group[0];
-            var secondary = group[1];
-
-            // There is 1 minute of time f or the nodes to connect to each other.
-            // (Should be easy, they're in the same pod.)
-            Time.Sleep(TimeSpan.FromMinutes(6));
-            bootstrapNode.BringOffline();
-
-            var file = GenerateTestFile(10.MB());
-            var contentId = primary.UploadFile(file);
-            var downloadedFile = secondary.DownloadContent(contentId);
-
-            file.AssertIsEqual(downloadedFile);
-        }
-
-        [Test]
-        public void DataRetentionTest()
-        {
-            var bootstrapNode = SetupCodexBootstrapNode(s => s.WithLogLevel(CodexLogLevel.Trace));
-
-            var startGroup = SetupCodexNodes(2, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode));
-            var finishGroup = SetupCodexNodes(10, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode));
-
-            var file = GenerateTestFile(10.MB());
-
-            // Both nodes in the start group have the file.
-            var content = startGroup[0].UploadFile(file);
-            DownloadAndAssert(content, file, startGroup[1]);
-
-            // Three nodes of the finish group have the file.
-            DownloadAndAssert(content, file, finishGroup[0]);
-            DownloadAndAssert(content, file, finishGroup[1]);
-            DownloadAndAssert(content, file, finishGroup[2]);
-
-            // The start group goes away.
-            startGroup.BringOffline();
-
-            // All nodes in the finish group can access the file.
-            foreach (var node in finishGroup)
-            {
-                DownloadAndAssert(content, file, node);
-            }
-        }
-
-        private void DownloadAndAssert(ContentId content, TestFile file, IOnlineCodexNode onlineCodexNode)
-        {
-            var downloaded = onlineCodexNode.DownloadContent(content);
-            file.AssertIsEqual(downloaded);
-        }
-    }
-}

Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs

@@ -0,0 +1,62 @@
+using DistTestCore;
+using DistTestCore.Helpers;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.PeerDiscoveryTests
+{
+    [TestFixture]
+    public class LayeredDiscoveryTests : DistTest
+    {
+        [Test]
+        public void TwoLayersTest()
+        {
+            var root = SetupCodexNode();
+            var l1Source = SetupCodexNode(s => s.WithBootstrapNode(root));
+            var l1Node = SetupCodexNode(s => s.WithBootstrapNode(root));
+            var l2Target = SetupCodexNode(s => s.WithBootstrapNode(l1Node));
+
+            AssertAllNodesConnected();
+        }
+
+        [Test]
+        public void ThreeLayersTest()
+        {
+            var root = SetupCodexNode();
+            var l1Source = SetupCodexNode(s => s.WithBootstrapNode(root));
+            var l1Node = SetupCodexNode(s => s.WithBootstrapNode(root));
+            var l2Node = SetupCodexNode(s => s.WithBootstrapNode(l1Node));
+            var l3Target = SetupCodexNode(s => s.WithBootstrapNode(l2Node));
+
+            AssertAllNodesConnected();
+        }
+
+        [TestCase(3)]
+        [TestCase(5)]
+        [TestCase(10)]
+        [TestCase(20)]
+        [TestCase(50)]
+        public void NodeChainTest(int chainLength)
+        {
+            var node = SetupCodexNode();
+            for (var i = 1; i < chainLength; i++)
+            {
+                node = SetupCodexNode(s => s.WithBootstrapNode(node));
+            }
+
+            AssertAllNodesConnected();
+
+            for (int i = 0; i < 5; i++)
+            {
+                Time.Sleep(TimeSpan.FromSeconds(30));
+                AssertAllNodesConnected();
+            }
+        }
+
+        private void AssertAllNodesConnected()
+        {
+            PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
+            //PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes());
+        }
+    }
+}

Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -0,0 +1,73 @@
+using DistTestCore;
+using DistTestCore.Helpers;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.PeerDiscoveryTests
+{
+    [TestFixture]
+    public class PeerDiscoveryTests : AutoBootstrapDistTest
+    {
+        [Test]
+        public void CanReportUnknownPeerId()
+        {
+            var unknownId = "16Uiu2HAkv2CHWpff3dj5iuVNERAp8AGKGNgpGjPexJZHSqUstfsK";
+            var node = SetupCodexNode();
+
+            var result = node.GetDebugPeer(unknownId);
+            Assert.That(result.IsPeerFound, Is.False);
+        }
+
+        [TestCase(2)]
+        [TestCase(3)]
+        [TestCase(10)]
+        public void VariableNodes(int number)
+        {
+            SetupCodexNodes(number);
+
+            AssertAllNodesConnected();
+        }
+
+        [TestCase(2)]
+        [TestCase(3)]
+        [TestCase(10)]
+        [TestCase(20)]
+        public void VariableNodesInPods(int number)
+        {
+            for (var i = 0; i < number; i++)
+            {
+                SetupCodexNode();
+            }
+
+            AssertAllNodesConnected();
+        }
+
+        [TestCase(3, 3)]
+        [TestCase(3, 5)]
+        [TestCase(3, 10)]
+        [TestCase(5, 10)]
+        [TestCase(3, 20)]
+        [TestCase(5, 20)]
+        public void StagedVariableNodes(int numberOfNodes, int numberOfStages)
+        {
+            for (var i = 0; i < numberOfStages; i++)
+            {
+                SetupCodexNodes(numberOfNodes);
+
+                AssertAllNodesConnected();
+            }
+
+            for (int i = 0; i < 5; i++)
+            {
+                Time.Sleep(TimeSpan.FromSeconds(30));
+                AssertAllNodesConnected();
+            }
+        }
+
+        private void AssertAllNodesConnected()
+        {
+            PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
+            //PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes());
+        }
+    }
+}

Utils/Time.cs

@@ -22,6 +22,11 @@
             result += $"{d.Seconds} secs";
             return result;
         }
+        
+        public static void WaitUntil(Func<bool> predicate)
+        {
+            WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
+        }
 
         public static void WaitUntil(Func<bool> predicate, TimeSpan timeout, TimeSpan retryTime)
         {
@@ -38,5 +43,74 @@
                 state = predicate();
             }
         }
+
+        public static void Retry(Action action, string description)
+        {
+            Retry(action, TimeSpan.FromMinutes(1), description);
+        }
+
+        public static T Retry<T>(Func<T> action, string description)
+        {
+            return Retry(action, TimeSpan.FromMinutes(1), description);
+        }
+
+        public static void Retry(Action action, TimeSpan timeout, string description)
+        {
+            Retry(action, timeout, TimeSpan.FromSeconds(1), description);
+        }
+
+        public static T Retry<T>(Func<T> action, TimeSpan timeout, string description)
+        {
+            return Retry(action, timeout, TimeSpan.FromSeconds(1), description);
+        }
+
+        public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime, string description)
+        {
+            var start = DateTime.UtcNow;
+            var exceptions = new List<Exception>();
+            while (true)
+            {
+                if (DateTime.UtcNow - start > timeout)
+                {
+                    throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
+                }
+
+                try
+                {
+                    action();
+                    return;
+                }
+                catch (Exception ex)
+                {
+                    exceptions.Add(ex);
+                }
+
+                Sleep(retryTime);
+            }
+        }
+
+        public static T Retry<T>(Func<T> action, TimeSpan timeout, TimeSpan retryTime, string description)
+        {
+            var start = DateTime.UtcNow;
+            var exceptions = new List<Exception>();
+            while (true)
+            {
+                if (DateTime.UtcNow - start > timeout)
+                {
+                    throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
+                }
+
+                try
+                {
+                    return action();
+                }
+                catch (Exception ex)
+                {
+                    exceptions.Add(ex);
+                }
+
+                Sleep(retryTime);
+            }
+        }
     }
 }

codexnode-manifest.yml

@@ -24,7 +24,7 @@ spec:
     spec:
       containers:
       - name: codex-node1
-        image: thatbenbierens/nim-codex:sha-b204837
+        image: codexstorage/nim-codex:sha-7b88ea0
         ports:
         - containerPort: 8080
           name: api-1
@@ -38,7 +38,7 @@ spec:
         - name: LISTEN_ADDRS
           value: "/ip4/0.0.0.0/tcp/8082"
       - name: codex-node2
-        image: thatbenbierens/nim-codex:sha-b204837
+        image: codexstorage/nim-codex:sha-7b88ea0
         ports:
         - containerPort: 8083
           name: api-2

docker/job.yaml

@@ -0,0 +1,48 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: ${NAMEPREFIX}-${RUNID}
+  namespace: ${NAMESPACE}
+  labels:
+    name: ${NAMEPREFIX}-${RUNID}
+    run-id: ${RUNID}
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      name: ${NAMEPREFIX}
+    spec:
+      containers:
+      - name: ${NAMEPREFIX}-runner
+        image: codexstorage/cs-codex-dist-tests:sha-300b91e
+        env:
+        - name: RUNNERLOCATION
+          value: InternalToCluster
+        - name: KUBECONFIG
+          value: /opt/kubeconfig.yaml
+        - name: LOGPATH
+          value: /var/log/cs-codex-dist-tests
+        - name: NAMESPACE
+          value: ${NAMESPACE}
+        - name: BRANCH
+          value: ${BRANCH}
+        - name: SOURCE
+          value: ${SOURCE}
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /opt/kubeconfig.yaml
+          subPath: kubeconfig.yaml
+        - name: logs
+          mountPath: /var/log/cs-codex-dist-tests
+        # command:
+        # - "dotnet"
+        # - "test"
+        # - "Tests"
+      restartPolicy: Never
+      volumes:
+        - name: kubeconfig
+          secret:
+            secretName: cs-codex-dist-tests-app-kubeconfig
+        - name: logs
+          hostPath:
+            path: /var/log/cs-codex-dist-tests