Merge branch 'feature/parallel-tests'

DistTestCore/Codex/CodexAccess.cs

@@ -6,10 +6,12 @@ namespace DistTestCore.Codex
     public class CodexAccess
     {
         private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
 
-        public CodexAccess(BaseLog log, RunningContainer runningContainer)
+        public CodexAccess(BaseLog log, ITimeSet timeSet, RunningContainer runningContainer)
         {
             this.log = log;
+            this.timeSet = timeSet;
             Container = runningContainer;
         }
 
@@ -40,11 +42,29 @@ namespace DistTestCore.Codex
             return Http().HttpPostJson($"storage/request/{contentId}", request);
         }
 
+        public void EnsureOnline()
+        {
+            try
+            {
+                var debugInfo = GetDebugInfo();
+                if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
+
+                var nodePeerId = debugInfo.id;
+                var nodeName = Container.Name;
+                log.AddStringReplace(nodePeerId, $"___{nodeName}___");
+            }
+            catch (Exception e)
+            {
+                log.Error($"Failed to start codex node: {e}. Test infra failure.");
+                throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
+            }
+        }
+
         private Http Http()
         {
             var ip = Container.Pod.Cluster.IP;
             var port = Container.ServicePorts[0].Number;
-            return new Http(log, ip, port, baseUrl: "/api/codex/v1");
+            return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1");
         }
 
         public string ConnectToPeer(string peerId, string peerMultiAddress)

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -1,12 +1,17 @@
-using DistTestCore.Marketplace;
+using System.Runtime.InteropServices;
+using DistTestCore.Marketplace;
 using KubernetesWorkflow;
 
 namespace DistTestCore.Codex
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        //public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635";
+        #if Arm64
-        public const string DockerImage = "thatbenbierens/codexlocal:latest";
+            public const string DockerImage = "emizzle/nim-codex-arm64:sha-c7af585";
+        #else
+			//public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635";
+            public const string DockerImage = "thatbenbierens/codexlocal:latest";
+        #endif
         public const string MetricsPortTag = "metrics_port";
 
         protected override string Image => DockerImage;

DistTestCore/CodexNodeGroup.cs

@@ -62,29 +62,15 @@ namespace DistTestCore
             return $"group:[{Containers.Describe()}]";
         }
 
-        private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
+        public void EnsureOnline()
         {
-            var access = new CodexAccess(lifecycle.Log, c);
+            foreach (var node in Nodes) node.CodexAccess.EnsureOnline();
-            EnsureOnline(access);
-            return factory.CreateOnlineCodexNode(access, this);
         }
 
-        private void EnsureOnline(CodexAccess access)
+        private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
         {
-            try
+            var access = new CodexAccess(lifecycle.Log, lifecycle.TimeSet, c);
-            {
+            return factory.CreateOnlineCodexNode(access, this);
-                var debugInfo = access.GetDebugInfo();
-                if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
-
-                var nodePeerId = debugInfo.id;
-                var nodeName = access.Container.Name;
-                lifecycle.Log.AddStringReplace(nodePeerId, $"___{nodeName}___");
-            }
-            catch (Exception e)
-            {
-                lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure.");
-                throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
-            }
         }
     }
 }

DistTestCore/CodexStarter.cs

@@ -44,7 +44,7 @@ namespace DistTestCore
         public void DeleteAllResources()
         {
             var workflow = CreateWorkflow();
-            workflow.DeleteAllResources();
+            workflow.DeleteTestResources();
 
             RunningGroups.Clear();
         }
@@ -74,6 +74,7 @@ namespace DistTestCore
         {
             var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
             RunningGroups.Add(group);
+            group.EnsureOnline();
             return group;
         }
 

DistTestCore/Configuration.cs

@@ -4,13 +4,13 @@ namespace DistTestCore
 {
     public class Configuration
     {
-        public KubernetesWorkflow.Configuration GetK8sConfiguration()
+        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
         {
             return new KubernetesWorkflow.Configuration(
-                k8sNamespace: "codex-test-ns",
+                k8sNamespacePrefix: "ct-",
                 kubeConfigFile: null,
-                operationTimeout: Timing.K8sOperationTimeout(),
+                operationTimeout: timeSet.K8sOperationTimeout(),
-                retryDelay: Timing.K8sServiceDelay(),
+                retryDelay: timeSet.WaitForK8sServiceDelay(),
                 locationMap: new[]
                 {
                     new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),

DistTestCore/DistTest.cs

@@ -6,23 +6,25 @@ using KubernetesWorkflow;
 using Logging;
 using NUnit.Framework;
 using System.Reflection;
-using Utils;
 
 namespace DistTestCore
 {
     [SetUpFixture]
+    [Parallelizable(ParallelScope.All)]
     public abstract class DistTest
     {
         private readonly Configuration configuration = new Configuration();
         private readonly Assembly[] testAssemblies;
-        private FixtureLog fixtureLog = null!;
+        private readonly FixtureLog fixtureLog;
-        private TestLifecycle lifecycle = null!;
+        private readonly object lifecycleLock = new object();
-        private DateTime testStart = DateTime.MinValue;
+        private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
 
         public DistTest()
         {
             var assemblies = AppDomain.CurrentDomain.GetAssemblies();
             testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
+
+            fixtureLog = new FixtureLog(configuration.GetLogConfig());
         }
 
         [OneTimeSetUp]
@@ -30,14 +32,11 @@ namespace DistTestCore
         {
             // Previous test run may have been interrupted.
             // Begin by cleaning everything up.
-            Timing.UseLongTimeouts = false;
-            fixtureLog = new FixtureLog(configuration.GetLogConfig());
-
             try
             {
                 Stopwatch.Measure(fixtureLog, "Global setup", () =>
                 {
-                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration());
+                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
                     wc.CreateWorkflow().DeleteAllResources();
                 });                
             }
@@ -57,8 +56,6 @@ namespace DistTestCore
         [SetUp]
         public void SetUpDistTest()
         {
-            Timing.UseLongTimeouts = ShouldUseLongTimeouts();
-
             if (GlobalTestFailure.HasFailed)
             {
                 Assert.Inconclusive("Skip test: Previous test failed during clean up.");
@@ -85,7 +82,7 @@ namespace DistTestCore
 
         public TestFile GenerateTestFile(ByteSize size)
         {
-            return lifecycle.FileManager.GenerateTestFile(size);
+            return Get().FileManager.GenerateTestFile(size);
         }
 
         public IOnlineCodexNode SetupCodexBootstrapNode()
@@ -128,12 +125,58 @@ namespace DistTestCore
 
         public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
         {
-            return lifecycle.CodexStarter.BringOnline((CodexSetup)codexSetup);
+            return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
         }
 
-        protected BaseLog Log
+        protected void Log(string msg)
         {
-            get { return lifecycle.Log; }
+            TestContext.Progress.WriteLine(msg);
+            Get().Log.Log(msg);
+        }
+
+        protected void Debug(string msg)
+        {
+            TestContext.Progress.WriteLine(msg);
+            Get().Log.Debug(msg);
+        }
+
+        private TestLifecycle Get()
+        {
+            lock (lifecycleLock)
+            {
+                return lifecycles[GetCurrentTestName()];
+            }
+        }
+
+        private void CreateNewTestLifecycle()
+        {
+            var testName = GetCurrentTestName();
+            Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
+            {
+                lock (lifecycleLock)
+                {
+                    lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
+                }
+            });
+        }
+
+        private void DisposeTestLifecycle()
+        {
+            var lifecycle = Get();
+            fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
+            Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
+            {
+                lifecycle.Log.EndTest();
+                IncludeLogsAndMetricsOnTestFailure(lifecycle);
+                lifecycle.DeleteAllResources();
+                lifecycle = null!;
+            });
+        }
+
+        private ITimeSet GetTimeSet()
+        {
+            if (ShouldUseLongTimeouts()) return new LongTimeSet();
+            return new DefaultTimeSet();
         }
 
         private bool ShouldUseLongTimeouts()
@@ -151,28 +194,7 @@ namespace DistTestCore
             return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
         }
 
-        private void CreateNewTestLifecycle()
+        private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
-        {
-            Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () =>
-            {
-                lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration);
-                testStart = DateTime.UtcNow;
-            });
-        }
-
-        private void DisposeTestLifecycle()
-        {
-            fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({GetTestDuration()})");
-            Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
-            {
-                lifecycle.Log.EndTest();
-                IncludeLogsAndMetricsOnTestFailure();
-                lifecycle.DeleteAllResources();
-                lifecycle = null!;
-            });
-        }
-
-        private void IncludeLogsAndMetricsOnTestFailure()
         {
             var result = TestContext.CurrentContext.Result;
             if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
@@ -182,8 +204,8 @@ namespace DistTestCore
                 if (IsDownloadingLogsAndMetricsEnabled())
                 {
                     lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
-                    DownloadAllLogs();
+                    DownloadAllLogs(lifecycle);
-                    DownloadAllMetrics();
+                    DownloadAllMetrics(lifecycle);
                 }
                 else
                 {
@@ -192,25 +214,19 @@ namespace DistTestCore
             }
         }
 
-        private string GetTestDuration()
+        private void DownloadAllLogs(TestLifecycle lifecycle)
         {
-            var testDuration = DateTime.UtcNow - testStart;
+            OnEachCodexNode(lifecycle, node =>
-            return Time.FormatDuration(testDuration);
-        }
-
-        private void DownloadAllLogs()
-        {
-            OnEachCodexNode(node =>
             {
                 lifecycle.DownloadLog(node);
             });
         }
 
-        private void DownloadAllMetrics()
+        private void DownloadAllMetrics(TestLifecycle lifecycle)
         {
             var metricsDownloader = new MetricsDownloader(lifecycle.Log);
 
-            OnEachCodexNode(node =>
+            OnEachCodexNode(lifecycle, node =>
             {
                 var m = node.Metrics as MetricsAccess;
                 if (m != null)
@@ -220,7 +236,7 @@ namespace DistTestCore
             });
         }
 
-        private void OnEachCodexNode(Action<OnlineCodexNode> action)
+        private void OnEachCodexNode(TestLifecycle lifecycle, Action<OnlineCodexNode> action)
         {
             var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
             foreach (var node in allNodes)

DistTestCore/DistTestCore.csproj

@@ -1,10 +1,14 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 	  <RootNamespace>DistTestCore</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>
+    <IsArm64 Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)' == 'Arm64'">true</IsArm64>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(IsArm64)'=='true'">
+    <DefineConstants>Arm64</DefineConstants>
   </PropertyGroup>
 
   <ItemGroup>

DistTestCore/FileManager.cs

@@ -1,5 +1,6 @@
 using Logging;
 using NUnit.Framework;
+using Utils;
 
 namespace DistTestCore
 {
@@ -13,13 +14,14 @@ namespace DistTestCore
     public class FileManager : IFileManager
     {
         public const int ChunkSize = 1024 * 1024;
+        private static NumberSource folderNumberSource = new NumberSource(0);
         private readonly Random random = new Random();
         private readonly TestLog log;
         private readonly string folder;
 
         public FileManager(TestLog log, Configuration configuration)
         {
-            folder = configuration.GetFileManagerFolder();
+            folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5"));
 
             EnsureDirectory();
             this.log = log;

DistTestCore/GethStarter.cs

@@ -36,13 +36,8 @@ namespace DistTestCore
             var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log);
             var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
 
-            foreach (var account in companionNode.Accounts)
+            var accounts = companionNode.Accounts.Select(a => a.Account).ToArray();
-            {
+            interaction.MintTestTokens(accounts, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
-                interaction.TransferWeiTo(account.Account, marketplaceConfig.InitialEth.Wei);
-                interaction.MintTestTokens(account.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
-            }
-
-            interaction.WaitForAllTransactions();
         }
 
         private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)

DistTestCore/Http.cs

@@ -10,13 +10,15 @@ namespace DistTestCore
     public class Http
     {
         private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
         private readonly string ip;
         private readonly int port;
         private readonly string baseUrl;
 
-        public Http(BaseLog log, string ip, int port, string baseUrl)
+        public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl)
         {
             this.log = log;
+            this.timeSet = timeSet;
             this.ip = ip;
             this.port = port;
             this.baseUrl = baseUrl;
@@ -103,7 +105,7 @@ namespace DistTestCore
             log.Debug($"({url}) = '{message}'", 3);
         }
 
-        private static T Retry<T>(Func<T> operation)
+        private T Retry<T>(Func<T> operation)
         {
             var retryCounter = 0;
 
@@ -115,9 +117,9 @@ namespace DistTestCore
                 }
                 catch (Exception exception)
                 {
-                    Timing.HttpCallRetryDelay();
+                    timeSet.HttpCallRetryDelay();
                     retryCounter++;
-                    if (retryCounter > Timing.HttpCallRetryCount())
+                    if (retryCounter > timeSet.HttpCallRetryCount())
                     {
                         Assert.Fail(exception.ToString());
                         throw;
@@ -140,10 +142,10 @@ namespace DistTestCore
             }
         }
 
-        private static HttpClient GetClient()
+        private HttpClient GetClient()
         {
             var client = new HttpClient();
-            client.Timeout = Timing.HttpCallTimeout();
+            client.Timeout = timeSet.HttpCallTimeout();
             return client;
         }
     }

DistTestCore/Marketplace/CodexContractsContainerRecipe.cs

@@ -4,7 +4,11 @@ namespace DistTestCore.Marketplace
 {
     public class CodexContractsContainerRecipe : ContainerRecipeFactory
     {
-        public const string DockerImage = "thatbenbierens/codex-contracts-deployment";
+        #if Arm64
+            public const string DockerImage = "emizzle/codex-contracts-deployment:latest";
+        #else
+            public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint";
+        #endif
         public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json";
         public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json";
 

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -19,13 +19,14 @@ namespace DistTestCore.Marketplace
             this.container = container;
         }
 
-        public string ExtractAccount(int? orderNumber)
+        public AllGethAccounts ExtractAccounts()
         {
             log.Debug();
-            var account = Retry(() => FetchAccount(orderNumber));
+            var accountsCsv = Retry(() => FetchAccountsCsv());
-            if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure.");
+            if (string.IsNullOrEmpty(accountsCsv)) throw new InvalidOperationException("Unable to fetch accounts.csv for geth node. Test infra failure.");
 
-            return account;
+            var lines = accountsCsv.Split('\n');
+            return new AllGethAccounts(lines.Select(ParseLineToAccount).ToArray());
         }
 
         public string ExtractPubKey()
@@ -37,15 +38,6 @@ namespace DistTestCore.Marketplace
             return pubKey;
         }
 
-        public string ExtractPrivateKey(int? orderNumber)
-        {
-            log.Debug();
-            var privKey = Retry(() => FetchPrivateKey(orderNumber));
-            if (string.IsNullOrEmpty(privKey)) throw new InvalidOperationException("Unable to fetch private key from geth node. Test infra failure.");
-
-            return privKey;
-        }
-
         public string ExtractMarketplaceAddress()
         {
             log.Debug();
@@ -88,14 +80,9 @@ namespace DistTestCore.Marketplace
             }
         }
 
-        private string FetchAccount(int? orderNumber)
+        private string FetchAccountsCsv()
         {
-            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetAccountFilename(orderNumber));
+            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename);
-        }
-
-        private string FetchPrivateKey(int? orderNumber)
-        {
-            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetPrivateKeyFilename(orderNumber));
         }
 
         private string FetchMarketplaceAddress()
@@ -120,6 +107,15 @@ namespace DistTestCore.Marketplace
             workflow.DownloadContainerLog(container, enodeFinder);
             return enodeFinder.GetPubKey();
         }
+
+        private GethAccount ParseLineToAccount(string l)
+        {
+            var tokens = l.Replace("\r", "").Split(',');
+            if (tokens.Length != 2) throw new InvalidOperationException();
+            var account = tokens[0];
+            var privateKey = tokens[1];
+            return new GethAccount(account, privateKey);
+        }
     }
 
     public class PubKeyFinder : LogHandler, ILogHandler

DistTestCore/Marketplace/GethBootstrapNodeInfo.cs

@@ -6,19 +6,19 @@ namespace DistTestCore.Marketplace
 {
     public class GethBootstrapNodeInfo
     {
-        public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string pubKey, string privateKey, Port discoveryPort)
+        public GethBootstrapNodeInfo(RunningContainers runningContainers, AllGethAccounts allAccounts, string pubKey, Port discoveryPort)
         {
             RunningContainers = runningContainers;
-            Account = account;
+            AllAccounts = allAccounts;
+            Account = allAccounts.Accounts[0];
             PubKey = pubKey;
-            PrivateKey = privateKey;
             DiscoveryPort = discoveryPort;
         }
 
         public RunningContainers RunningContainers { get; }
-        public string Account { get; }
+        public AllGethAccounts AllAccounts { get; }
+        public GethAccount Account { get; }
         public string PubKey { get; }
-        public string PrivateKey { get; }
         public Port DiscoveryPort { get; }
 
         public NethereumInteraction StartInteraction(BaseLog log)
@@ -26,10 +26,19 @@ namespace DistTestCore.Marketplace
             var ip = RunningContainers.RunningPod.Cluster.IP;
             var port = RunningContainers.Containers[0].ServicePorts[0].Number;
             var account = Account;
-            var privateKey = PrivateKey;
 
-            var creator = new NethereumInteractionCreator(log, ip, port, account, privateKey);
+            var creator = new NethereumInteractionCreator(log, ip, port, account.PrivateKey);
             return creator.CreateWorkflow();
         }
     }
+
+    public class AllGethAccounts
+    {
+        public GethAccount[] Accounts { get; }
+
+        public AllGethAccounts(GethAccount[] accounts)
+        {
+            Accounts = accounts;
+        }
+    }
 }

DistTestCore/Marketplace/GethBootstrapNodeStarter.cs

@@ -20,20 +20,20 @@ namespace DistTestCore.Marketplace
             var bootstrapContainer = containers.Containers[0];
 
             var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer);
-            var account = extractor.ExtractAccount(null);
+            var accounts = extractor.ExtractAccounts();
             var pubKey = extractor.ExtractPubKey();
-            var privateKey = extractor.ExtractPrivateKey(null);
             var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
+            var result = new GethBootstrapNodeInfo(containers, accounts, pubKey, discoveryPort);
 
-            LogEnd($"Geth bootstrap node started with account '{account}'");
+            LogEnd($"Geth bootstrap node started with account '{result.Account.Account}'");
 
-            return new GethBootstrapNodeInfo(containers, account, pubKey, privateKey, discoveryPort);
+            return result;
         }
 
         private StartupConfig CreateBootstrapStartupConfig()
         {
             var config = new StartupConfig();
-            config.Add(new GethStartupConfig(true, null!, 0));
+            config.Add(new GethStartupConfig(true, null!, 0, 0));
             return config;
         }
     }

DistTestCore/Marketplace/GethCompanionNodeInfo.cs

@@ -6,30 +6,29 @@ namespace DistTestCore.Marketplace
 {
     public class GethCompanionNodeInfo
     {
-        public GethCompanionNodeInfo(RunningContainer runningContainer, GethCompanionAccount[] accounts)
+        public GethCompanionNodeInfo(RunningContainer runningContainer, GethAccount[] accounts)
         {
             RunningContainer = runningContainer;
             Accounts = accounts;
         }
 
         public RunningContainer RunningContainer { get; }
-        public GethCompanionAccount[] Accounts { get; }
+        public GethAccount[] Accounts { get; }
         
-        public NethereumInteraction StartInteraction(BaseLog log, GethCompanionAccount account)
+        public NethereumInteraction StartInteraction(BaseLog log, GethAccount account)
         {
             var ip = RunningContainer.Pod.Cluster.IP;
             var port = RunningContainer.ServicePorts[0].Number;
-            var accountStr = account.Account;
             var privateKey = account.PrivateKey;
 
-            var creator = new NethereumInteractionCreator(log, ip, port, accountStr, privateKey);
+            var creator = new NethereumInteractionCreator(log, ip, port, privateKey);
             return creator.CreateWorkflow();
         }
     }
 
-    public class GethCompanionAccount
+    public class GethAccount
     {
-        public GethCompanionAccount(string account, string privateKey)
+        public GethAccount(string account, string privateKey)
         {
             Account = account;
             PrivateKey = privateKey;

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -5,6 +5,8 @@ namespace DistTestCore.Marketplace
 {
     public class GethCompanionNodeStarter : BaseStarter
     {
+        private int companionAccountIndex = 0;
+
         public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
             : base(lifecycle, workflowCreator)
         {
@@ -14,53 +16,43 @@ namespace DistTestCore.Marketplace
         {
             LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes.");
 
-            var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
+            var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
 
             var workflow = workflowCreator.CreateWorkflow();
-            var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
-            WaitForAccountCreation(codexSetup.NumberOfNodes);
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
             var container = containers.Containers[0];
 
-            var node = CreateCompanionInfo(workflow, container, codexSetup.NumberOfNodes);
+            var node = CreateCompanionInfo(container, marketplace, config);
             EnsureCompanionNodeIsSynced(node, marketplace);
 
             LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]");
             return node;
         }
 
-        private void WaitForAccountCreation(int numberOfNodes)
+        private GethCompanionNodeInfo CreateCompanionInfo(RunningContainer container, MarketplaceNetwork marketplace, GethStartupConfig config)
         {
-            // We wait proportional to the number of account the node has to create. It takes a few seconds for each one to generate the keys and create the files
+            var accounts = ExtractAccounts(marketplace, config);
-            // we will be trying to read in 'ExtractAccount', later on in the start-up process.
-            Time.Sleep(TimeSpan.FromSeconds(4.5 * numberOfNodes));
-        }
-
-        private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container, int numberOfAccounts)
-        {
-            var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
-            var accounts = ExtractAccounts(extractor, numberOfAccounts).ToArray();
             return new GethCompanionNodeInfo(container, accounts);
         }
 
-        private IEnumerable<GethCompanionAccount> ExtractAccounts(ContainerInfoExtractor extractor, int numberOfAccounts)
+        private static GethAccount[] ExtractAccounts(MarketplaceNetwork marketplace, GethStartupConfig config)
         {
-            for (int i = 0; i < numberOfAccounts; i++) yield return ExtractAccount(extractor, i + 1);
+            return marketplace.Bootstrap.AllAccounts.Accounts
-        }
+                .Skip(1 + config.CompanionAccountStartIndex)
-
+                .Take(config.NumberOfCompanionAccounts)
-        private GethCompanionAccount ExtractAccount(ContainerInfoExtractor extractor, int orderNumber)
+                .ToArray();
-        {
-            var account = extractor.ExtractAccount(orderNumber);
-            var privKey = extractor.ExtractPrivateKey(orderNumber);
-            return new GethCompanionAccount(account, privKey);
         }
 
         private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
         {
             try
             {
-                var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
+                Time.WaitUntil(() =>
-                interaction.EnsureSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
+                {
+                    var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
+                    return interaction.IsSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
+                }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
             }
             catch (Exception e)
             {
@@ -68,10 +60,17 @@ namespace DistTestCore.Marketplace
             }
         }
 
-        private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
+        private GethStartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
+        {
+            var config = new GethStartupConfig(false, bootstrapNode, companionAccountIndex, numberOfAccounts);
+            companionAccountIndex += numberOfAccounts;
+            return config;
+        }
+
+        private StartupConfig CreateStartupConfig(GethStartupConfig gethConfig)
         {
             var config = new StartupConfig();
-            config.Add(new GethStartupConfig(false, bootstrapNode, numberOfAccounts));
+            config.Add(gethConfig);
             return config;
         }
     }

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -4,22 +4,17 @@ namespace DistTestCore.Marketplace
 {
     public class GethContainerRecipe : ContainerRecipeFactory
     {
-        public const string DockerImage = "thatbenbierens/geth-confenv:latest";
+        #if Arm64
+            public const string DockerImage = "emizzle/geth-confenv:latest";
+        #else
+            public const string DockerImage = "thatbenbierens/geth-confenv:onethousand";
+        #endif
+
         public const string HttpPortTag = "http_port";
         public const string DiscoveryPortTag = "disc_port";
         private const string defaultArgs = "--ipcdisable --syncmode full";
 
-        public static string GetAccountFilename(int? orderNumber)
+        public const string AccountsFilename = "accounts.csv";
-        {
-            if (orderNumber == null) return "account_string.txt";
-            return $"account_string_{orderNumber.Value}.txt";
-        }
-
-        public static string GetPrivateKeyFilename(int? orderNumber)
-        {
-            if (orderNumber == null) return "private.key";
-            return $"private_{orderNumber.Value}.key";
-        }
 
         protected override string Image => DockerImage;
 
@@ -46,14 +41,17 @@ namespace DistTestCore.Marketplace
 
         private string CreateBootstapArgs(Port discovery)
         {
-            AddEnvVar("IS_BOOTSTRAP", "1");
+            AddEnvVar("ENABLE_MINER", "1");
+            UnlockAccounts(0, 1);
             var exposedPort = AddExposedPort(tag: HttpPortTag);
             return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}";
         }
 
         private string CreateCompanionArgs(Port discovery, GethStartupConfig config)
         {
-            AddEnvVar("NUMBER_OF_ACCOUNTS", config.NumberOfCompanionAccounts.ToString());
+            UnlockAccounts(
+                config.CompanionAccountStartIndex + 1,
+                config.NumberOfCompanionAccounts);
 
             var port = AddInternalPort();
             var authRpc = AddInternalPort();
@@ -66,5 +64,15 @@ namespace DistTestCore.Marketplace
 
             return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}";
         }
+
+        private void UnlockAccounts(int startIndex, int numberOfAccounts)
+        {
+            if (startIndex < 0) throw new ArgumentException();
+            if (numberOfAccounts < 1) throw new ArgumentException();
+            if (startIndex + numberOfAccounts > 1000) throw new ArgumentException("Out of accounts!");
+
+            AddEnvVar("UNLOCK_START_INDEX", startIndex.ToString());
+            AddEnvVar("UNLOCK_NUMBER", numberOfAccounts.ToString());
+        }
     }
 }

DistTestCore/Marketplace/GethStartupConfig.cs

@@ -2,15 +2,17 @@
 {
     public class GethStartupConfig
     {
-        public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int numberOfCompanionAccounts)
+        public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int companionAccountStartIndex, int numberOfCompanionAccounts)
         {
             IsBootstrapNode = isBootstrapNode;
             BootstrapNode = bootstrapNode;
+            CompanionAccountStartIndex = companionAccountStartIndex;
             NumberOfCompanionAccounts = numberOfCompanionAccounts;
         }
 
         public bool IsBootstrapNode { get; }
         public GethBootstrapNodeInfo BootstrapNode { get; }
+        public int CompanionAccountStartIndex { get; }
         public int NumberOfCompanionAccounts { get; }
     }
 }

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -19,10 +19,10 @@ namespace DistTestCore.Marketplace
     {
         private readonly TestLog log;
         private readonly MarketplaceNetwork marketplaceNetwork;
-        private readonly GethCompanionAccount account;
+        private readonly GethAccount account;
         private readonly CodexAccess codexAccess;
 
-        public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionAccount account, CodexAccess codexAccess)
+        public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
         {
             this.log = log;
             this.marketplaceNetwork = marketplaceNetwork;

DistTestCore/Marketplace/MarketplaceAccessFactory.cs

@@ -33,10 +33,10 @@ namespace DistTestCore.Marketplace
             return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access);
         }
 
-        private GethCompanionAccount GetGethCompanionNode(CodexAccess access)
+        private GethAccount GetGethCompanionNode(CodexAccess access)
         {
-            var account = access.Container.Recipe.Additionals.Single(a => a is GethCompanionAccount);
+            var account = access.Container.Recipe.Additionals.Single(a => a is GethAccount);
-            return (GethCompanionAccount)account;
+            return (GethAccount)account;
         }
     }
 }

DistTestCore/Metrics/MetricsAccess.cs

@@ -14,12 +14,14 @@ namespace DistTestCore.Metrics
     public class MetricsAccess : IMetricsAccess
     {
         private readonly TestLog log;
+        private readonly ITimeSet timeSet;
         private readonly MetricsQuery query;
         private readonly RunningContainer node;
 
-        public MetricsAccess(TestLog log, MetricsQuery query, RunningContainer node)
+        public MetricsAccess(TestLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
         {
             this.log = log;
+            this.timeSet = timeSet;
             this.query = query;
             this.node = node;
         }
@@ -47,7 +49,7 @@ namespace DistTestCore.Metrics
             {
                 var mostRecent = GetMostRecent(metricName);
                 if (mostRecent != null) return mostRecent;
-                if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
+                if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout())
                 {
                     Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
                     throw new TimeoutException();

DistTestCore/Metrics/MetricsAccessFactory.cs

@@ -28,8 +28,8 @@ namespace DistTestCore.Metrics
 
         public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
         {
-            var query = new MetricsQuery(lifecycle.Log, prometheusContainer);
+            var query = new MetricsQuery(lifecycle.Log, lifecycle.TimeSet, prometheusContainer);
-            return new MetricsAccess(lifecycle.Log, query, codexContainer);
+            return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
         }
     }
 }

DistTestCore/Metrics/MetricsQuery.cs

@@ -9,12 +9,13 @@ namespace DistTestCore.Metrics
     {
         private readonly Http http;
 
-        public MetricsQuery(BaseLog log, RunningContainers runningContainers)
+        public MetricsQuery(BaseLog log, ITimeSet timeSet, RunningContainers runningContainers)
         {
             RunningContainers = runningContainers;
 
             http = new Http(
                 log,
+                timeSet,
                 runningContainers.RunningPod.Cluster.IP,
                 runningContainers.Containers[0].ServicePorts[0].Number,
                 "api/v1");

DistTestCore/OnlineCodexNode.cs

@@ -92,6 +92,10 @@ namespace DistTestCore
 
         public ICodexSetup BringOffline()
         {
+            if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
+                "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
+                "available for codex-nodes in groups of 1.");
+
             return Group.BringOffline();
         }
 

DistTestCore/TestLifecycle.cs

@@ -1,25 +1,30 @@
 using DistTestCore.Logs;
 using KubernetesWorkflow;
 using Logging;
+using Utils;
 
 namespace DistTestCore
 {
     public class TestLifecycle
     {
         private readonly WorkflowCreator workflowCreator;
+        private DateTime testStart = DateTime.MinValue;
 
-        public TestLifecycle(TestLog log, Configuration configuration)
+        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
         {
             Log = log;
-            workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration());
+            TimeSet = timeSet;
+            workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet));
 
             FileManager = new FileManager(Log, configuration);
             CodexStarter = new CodexStarter(this, workflowCreator);
             PrometheusStarter = new PrometheusStarter(this, workflowCreator);
             GethStarter = new GethStarter(this, workflowCreator);
+            testStart = DateTime.UtcNow;
         }
 
         public TestLog Log { get; }
+        public ITimeSet TimeSet { get; }
         public FileManager FileManager { get; }
         public CodexStarter CodexStarter { get; }
         public PrometheusStarter PrometheusStarter { get; }
@@ -42,5 +47,11 @@ namespace DistTestCore
 
             return new CodexNodeLog(subFile, node);
         }
+
+        public string GetTestDuration()
+        {
+            var testDuration = DateTime.UtcNow - testStart;
+            return Time.FormatDuration(testDuration);
+        }
     }
 }

DistTestCore/Timing.cs

@@ -8,53 +8,11 @@ namespace DistTestCore
     {
     }
 
-    public static class Timing
-    {
-        public static bool UseLongTimeouts { get; set; }
-
-
-        public static TimeSpan HttpCallTimeout()
-        {
-            return GetTimes().HttpCallTimeout();
-        }
-
-        public static int HttpCallRetryCount()
-        {
-            return GetTimes().HttpCallRetryCount();
-        }
-
-        public static void HttpCallRetryDelay()
-        {
-            Time.Sleep(GetTimes().HttpCallRetryDelay());
-        }
-
-        public static TimeSpan K8sServiceDelay()
-        {
-            return GetTimes().WaitForK8sServiceDelay();
-        }
-
-        public static TimeSpan K8sOperationTimeout()
-        {
-            return GetTimes().K8sOperationTimeout();
-        }
-
-        public static TimeSpan WaitForMetricTimeout()
-        {
-            return GetTimes().WaitForMetricTimeout();
-        }
-
-        private static ITimeSet GetTimes()
-        {
-            if (UseLongTimeouts) return new LongTimeSet();
-            return new DefaultTimeSet();
-        }
-    }
-
     public interface ITimeSet
     {
         TimeSpan HttpCallTimeout();
         int HttpCallRetryCount();
-        TimeSpan HttpCallRetryDelay();
+        void HttpCallRetryDelay();
         TimeSpan WaitForK8sServiceDelay();
         TimeSpan K8sOperationTimeout();
         TimeSpan WaitForMetricTimeout();
@@ -72,9 +30,9 @@ namespace DistTestCore
             return 5;
         }
 
-        public TimeSpan HttpCallRetryDelay()
+        public void HttpCallRetryDelay()
         {
-            return TimeSpan.FromSeconds(3);
+            Time.Sleep(TimeSpan.FromSeconds(3));
         }
 
         public TimeSpan WaitForK8sServiceDelay()
@@ -105,9 +63,9 @@ namespace DistTestCore
             return 2;
         }
 
-        public TimeSpan HttpCallRetryDelay()
+        public void HttpCallRetryDelay()
         {
-            return TimeSpan.FromMinutes(5);
+            Time.Sleep(TimeSpan.FromMinutes(5));
         }
 
         public TimeSpan WaitForK8sServiceDelay()

KubernetesWorkflow/ApplicationLifecycle.cs

@@ -0,0 +1,40 @@
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public class ApplicationLifecycle
+    {
+        private static object instanceLock = new object();
+        private static ApplicationLifecycle? instance;
+        private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
+        private readonly NumberSource namespaceNumberSource = new NumberSource(0);
+
+        private ApplicationLifecycle()
+        {
+        }
+
+        public static ApplicationLifecycle Instance
+        {
+            // I know singletons are quite evil. But we need to be sure this object is created only once
+            // and persists for the entire application lifecycle.
+            get
+            {
+                lock (instanceLock)
+                {
+                    if (instance == null) instance = new ApplicationLifecycle();
+                    return instance;
+                }
+            }
+        }
+
+        public NumberSource GetServiceNumberSource()
+        {
+            return servicePortNumberSource;
+        }
+
+        public string GetTestNamespace()
+        {
+            return namespaceNumberSource.GetNextNumber().ToString("D5");
+        }
+    }
+}

KubernetesWorkflow/CommandRunner.cs

@@ -5,7 +5,7 @@ namespace KubernetesWorkflow
 {
     public class CommandRunner
     {
-        private readonly Kubernetes client;
+        private readonly K8sClient client;
         private readonly string k8sNamespace;
         private readonly RunningPod pod;
         private readonly string containerName;
@@ -13,7 +13,7 @@ namespace KubernetesWorkflow
         private readonly string[] arguments;
         private readonly List<string> lines = new List<string>();
 
-        public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
+        public CommandRunner(K8sClient client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
         {
             this.client = client;
             this.k8sNamespace = k8sNamespace;
@@ -27,8 +27,8 @@ namespace KubernetesWorkflow
         {
             var input = new[] { command }.Concat(arguments).ToArray();
 
-            Time.Wait(client.NamespacedPodExecAsync(
+            Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
-                pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()));
+                pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
         }
 
         public string GetStdOut()

KubernetesWorkflow/Configuration.cs

@@ -2,16 +2,16 @@
 {
     public class Configuration
     {
-        public Configuration(string k8sNamespace, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
+        public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
         {
-            K8sNamespace = k8sNamespace;
+            K8sNamespacePrefix = k8sNamespacePrefix;
             KubeConfigFile = kubeConfigFile;
             OperationTimeout = operationTimeout;
             RetryDelay = retryDelay;
             LocationMap = locationMap;
         }
 
-        public string K8sNamespace { get; }
+        public string K8sNamespacePrefix { get; }
         public string? KubeConfigFile { get; }
         public TimeSpan OperationTimeout { get; }
         public TimeSpan RetryDelay { get; }

KubernetesWorkflow/ContainerRecipe.cs

@@ -24,6 +24,14 @@
         {
             return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag);
         }
+
+        public override string ToString()
+        {
+            return $"(container-recipe: {Name}, image: {Image}, " +
+                $"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " +
+                $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
+                $"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, ";
+        }
     }
 
     public class Port

KubernetesWorkflow/K8sClient.cs

@@ -0,0 +1,36 @@
+using k8s;
+
+namespace KubernetesWorkflow
+{
+    public class K8sClient
+    {
+        private readonly Kubernetes client;
+        private static readonly object clientLock = new object();
+
+        public K8sClient(KubernetesClientConfiguration config)
+        {
+            client = new Kubernetes(config);
+        }
+
+        public void Run(Action<Kubernetes> action)
+        {
+            lock (clientLock)
+            {
+                action(client);
+            }
+        }
+
+        public T Run<T>(Func<Kubernetes, T> action)
+        {
+            lock (clientLock)
+            {
+                return action(client);
+            }
+        }
+
+        public void Dispose()
+        {
+            client.Dispose();
+        }
+    }
+}

KubernetesWorkflow/K8sController.cs

@@ -11,16 +11,18 @@ namespace KubernetesWorkflow
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownPods;
         private readonly WorkflowNumberSource workflowNumberSource;
-        private readonly Kubernetes client;
+        private readonly K8sClient client;
 
-        public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource)
+        public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
         {
             this.log = log;
             this.cluster = cluster;
             this.knownPods = knownPods;
             this.workflowNumberSource = workflowNumberSource;
+            client = new K8sClient(cluster.GetK8sClientConfig());
 
-            client = new Kubernetes(cluster.GetK8sClientConfig());
+            K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
+            log.Debug($"Test namespace: '{K8sTestNamespace}'");
         }
 
         public void Dispose()
@@ -52,14 +54,14 @@ namespace KubernetesWorkflow
         public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
         {
             log.Debug();
-            using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name);
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.Name, K8sTestNamespace, recipe.Name));
             logHandler.Log(stream);
         }
 
         public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
         {
             log.Debug($"{containerName}: {command} ({string.Join(",", args)})");
-            var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
+            var runner = new CommandRunner(client, K8sTestNamespace, pod, containerName, command, args);
             runner.Run();
             return runner.GetStdOut();
         }
@@ -67,13 +69,43 @@ namespace KubernetesWorkflow
         public void DeleteAllResources()
         {
             log.Debug();
-            DeleteNamespace();
 
+            var all = client.Run(c => c.ListNamespace().Items);
+            var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(cluster.Configuration.K8sNamespacePrefix));
+
+            foreach (var ns in namespaces)
+            {
+                DeleteNamespace(ns);
+            }
+            foreach (var ns in namespaces)
+            {
+                WaitUntilNamespaceDeleted(ns);
+            }
+        }
+
+        public void DeleteTestNamespace()
+        {
+            log.Debug();
+            if (IsTestNamespaceOnline())
+            {
+                client.Run(c => c.DeleteNamespace(K8sTestNamespace, null, null, gracePeriodSeconds: 0));
+            }
             WaitUntilNamespaceDeleted();
         }
 
+        public void DeleteNamespace(string ns)
+        {
+            log.Debug();
+            if (IsNamespaceOnline(ns))
+            {
+                client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
+            }
+        }
+
         #region Namespace management
 
+        private string K8sTestNamespace { get; }
+
         private void EnsureTestNamespace()
         {
             if (IsTestNamespaceOnline()) return;
@@ -83,30 +115,85 @@ namespace KubernetesWorkflow
                 ApiVersion = "v1",
                 Metadata = new V1ObjectMeta
                 {
-                    Name = K8sNamespace,
+                    Name = K8sTestNamespace,
-                    Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
+                    Labels = new Dictionary<string, string> { { "name", K8sTestNamespace } }
                 }
             };
-            client.CreateNamespace(namespaceSpec);
+            client.Run(c => c.CreateNamespace(namespaceSpec));
             WaitUntilNamespaceCreated();
-        }
 
-        private void DeleteNamespace()
+            CreatePolicy();
-        {
-            if (IsTestNamespaceOnline())
-            {
-                client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0);
-            }
-        }
-
-        private string K8sNamespace
-        {
-            get { return cluster.Configuration.K8sNamespace; }
         }
 
         private bool IsTestNamespaceOnline()
         {
-            return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace);
+            return IsNamespaceOnline(K8sTestNamespace);
+        }
+
+        private bool IsNamespaceOnline(string name)
+        {
+            return client.Run(c => c.ListNamespace().Items.Any(n => n.Metadata.Name == name));
+        }
+
+        private void CreatePolicy()
+        {
+            client.Run(c =>
+            {
+                var body = new V1NetworkPolicy
+                {
+                    Metadata = new V1ObjectMeta
+                    {
+                        Name = "isolate-policy",
+                        NamespaceProperty = K8sTestNamespace
+                    },
+                    Spec = new V1NetworkPolicySpec
+                    {
+                        PodSelector = new V1LabelSelector
+                        {
+                            MatchLabels = GetSelector()
+                        },
+                        PolicyTypes = new[]
+                        {
+                            "Ingress",
+                            "Egress"
+                        },
+                        Ingress = new List<V1NetworkPolicyIngressRule>
+                        {
+                            new V1NetworkPolicyIngressRule
+                            {
+                                FromProperty = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        NamespaceSelector = new V1LabelSelector
+                                        {
+                                            MatchLabels = GetMyNamespaceSelector()
+                                        }
+                                    }
+                                }
+                            }
+                        },
+                        Egress = new List<V1NetworkPolicyEgressRule>
+                        {
+                            new V1NetworkPolicyEgressRule
+                            {
+                                To = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        NamespaceSelector = new V1LabelSelector
+                                        {
+                                            MatchLabels = GetMyNamespaceSelector()
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                };
+
+                c.CreateNamespacedNetworkPolicy(body, K8sTestNamespace);
+            });
         }
 
         #endregion
@@ -141,7 +228,7 @@ namespace KubernetesWorkflow
                 }
             };
 
-            client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
+            client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sTestNamespace));
             WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
 
             return deploymentSpec.Metadata.Name;
@@ -149,7 +236,7 @@ namespace KubernetesWorkflow
 
         private void DeleteDeployment(string deploymentName)
         {
-            client.DeleteNamespacedDeployment(deploymentName, K8sNamespace);
+            client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sTestNamespace));
             WaitUntilDeploymentOffline(deploymentName);
         }
 
@@ -168,12 +255,18 @@ namespace KubernetesWorkflow
             return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
         }
 
+        private IDictionary<string, string> GetMyNamespaceSelector()
+        {
+            return new Dictionary<string, string> { { "name", "thatisincorrect" } };
+        }
+
         private V1ObjectMeta CreateDeploymentMetadata()
         {
             return new V1ObjectMeta
             {
                 Name = "deploy-" + workflowNumberSource.WorkflowNumber,
-                NamespaceProperty = K8sNamespace
+                NamespaceProperty = K8sTestNamespace,
+                Labels = GetSelector()
             };
         }
 
@@ -257,14 +350,14 @@ namespace KubernetesWorkflow
                 }
             };
 
-            client.CreateNamespacedService(serviceSpec, K8sNamespace);
+            client.Run(c => c.CreateNamespacedService(serviceSpec, K8sTestNamespace));
 
             return (serviceSpec.Metadata.Name, result);
         }
 
         private void DeleteService(string serviceName)
         {
-            client.DeleteNamespacedService(serviceName, K8sNamespace);
+            client.Run(c => c.DeleteNamespacedService(serviceName, K8sTestNamespace));
         }
 
         private V1ObjectMeta CreateServiceMetadata()
@@ -272,7 +365,7 @@ namespace KubernetesWorkflow
             return new V1ObjectMeta
             {
                 Name = "service-" + workflowNumberSource.WorkflowNumber,
-                NamespaceProperty = K8sNamespace
+                NamespaceProperty = K8sTestNamespace
             };
         }
 
@@ -323,11 +416,16 @@ namespace KubernetesWorkflow
             WaitUntil(() => !IsTestNamespaceOnline());
         }
 
+        private void WaitUntilNamespaceDeleted(string name)
+        {
+            WaitUntil(() => !IsNamespaceOnline(name));
+        }
+
         private void WaitUntilDeploymentOnline(string deploymentName)
         {
             WaitUntil(() =>
             {
-                var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
+                var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sTestNamespace));
                 return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
             });
         }
@@ -336,7 +434,7 @@ namespace KubernetesWorkflow
         {
             WaitUntil(() =>
             {
-                var deployments = client.ListNamespacedDeployment(K8sNamespace);
+                var deployments = client.Run(c => c.ListNamespacedDeployment(K8sTestNamespace));
                 var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
                 return deployment == null || deployment.Status.AvailableReplicas == 0;
             });
@@ -346,7 +444,7 @@ namespace KubernetesWorkflow
         {
             WaitUntil(() =>
             {
-                var pods = client.ListNamespacedPod(K8sNamespace).Items;
+                var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
                 var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
                 return pod == null;
             });
@@ -369,7 +467,7 @@ namespace KubernetesWorkflow
 
         private (string, string) FetchNewPod()
         {
-            var pods = client.ListNamespacedPod(K8sNamespace).Items;
+            var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;
 
             var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
             if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");

KubernetesWorkflow/KubernetesWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 	  <RootNamespace>KubernetesWorkflow</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>

KubernetesWorkflow/StartupWorkflow.cs

@@ -8,14 +8,16 @@ namespace KubernetesWorkflow
         private readonly WorkflowNumberSource numberSource;
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownK8SPods;
+        private readonly string testNamespace;
         private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
 
-        internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods)
+        internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
         {
             this.log = log;
             this.numberSource = numberSource;
             this.cluster = cluster;
             this.knownK8SPods = knownK8SPods;
+            this.testNamespace = testNamespace;
         }
 
         public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
@@ -62,10 +64,24 @@ namespace KubernetesWorkflow
             });
         }
 
+        public void DeleteTestResources()
+        {
+            K8s(controller =>
+            {
+                controller.DeleteTestNamespace();
+            });
+        }
+
         private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig)
         {
             log.Debug();
-            return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r), startupConfig)).ToArray();
+            return recipes.Select(r =>
+            {
+                var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
+                log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
+
+                return new RunningContainer(runningPod, r, servicePorts, startupConfig);
+            }).ToArray();
         }
 
         private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
@@ -74,7 +90,7 @@ namespace KubernetesWorkflow
             var result = new List<ContainerRecipe>();
             for (var i = 0; i < numberOfContainers; i++)
             {
-                result.Add(recipeFactory.CreateRecipe(i ,numberSource.GetContainerNumber(), componentFactory, startupConfig));
+                result.Add(recipeFactory.CreateRecipe(i, numberSource.GetContainerNumber(), componentFactory, startupConfig));
             }
 
             return result.ToArray();
@@ -82,14 +98,14 @@ namespace KubernetesWorkflow
 
         private void K8s(Action<K8sController> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
             action(controller);
             controller.Dispose();
         }
 
         private T K8s<T>(Func<K8sController, T> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
             var result = action(controller);
             controller.Dispose();
             return result;

KubernetesWorkflow/WorkflowCreator.cs

@@ -6,25 +6,26 @@ namespace KubernetesWorkflow
     public class WorkflowCreator
     {
         private readonly NumberSource numberSource = new NumberSource(0);
-        private readonly NumberSource servicePortNumberSource = new NumberSource(30001);
         private readonly NumberSource containerNumberSource = new NumberSource(0);
         private readonly KnownK8sPods knownPods = new KnownK8sPods();
         private readonly K8sCluster cluster;
         private readonly BaseLog log;
+        private readonly string testNamespace;
 
         public WorkflowCreator(BaseLog log, Configuration configuration)
         {
             cluster = new K8sCluster(configuration);
             this.log = log;
+            testNamespace = ApplicationLifecycle.Instance.GetTestNamespace();
         }
 
         public StartupWorkflow CreateWorkflow()
         {
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
-                                                                    servicePortNumberSource,
+                                                                    ApplicationLifecycle.Instance.GetServiceNumberSource(),
                                                                     containerNumberSource);
 
-            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods);
+            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);
         }
     }
 }

Logging/Logging.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 	  <RootNamespace>Logging</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

LongTests/TestsLong.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Nethereum/NethereumInteraction.cs

@@ -10,16 +10,13 @@ namespace NethereumWorkflow
 {
     public class NethereumInteraction
     {
-        private readonly List<Task> openTasks = new List<Task>();
         private readonly BaseLog log;
         private readonly Web3 web3;
-        private readonly string rootAccount;
 
-        internal NethereumInteraction(BaseLog log, Web3 web3, string rootAccount)
+        internal NethereumInteraction(BaseLog log, Web3 web3)
         {
             this.log = log;
             this.web3 = web3;
-            this.rootAccount = rootAccount;
         }
 
         public string GetTokenAddress(string marketplaceAddress)
@@ -31,29 +28,13 @@ namespace NethereumWorkflow
             return Time.Wait(handler.QueryAsync<string>(marketplaceAddress, function));
         }
 
-        public void TransferWeiTo(string account, decimal amount)
+        public void MintTestTokens(string[] accounts, decimal amount, string tokenAddress)
         {
-            log.Debug($"{amount} --> {account}");
+            if (amount < 1 || accounts.Length < 1) throw new ArgumentException("Invalid arguments for MintTestTokens");
-            if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance");
 
-            var value = ToHexBig(amount);
+            var tasks = accounts.Select(a => MintTokens(a, amount, tokenAddress));
-            var transactionId = Time.Wait(web3.Eth.TransactionManager.SendTransactionAsync(rootAccount, account, value));
-            openTasks.Add(web3.Eth.TransactionManager.TransactionReceiptService.PollForReceiptAsync(transactionId));
-        }
 
-        public void MintTestTokens(string account, decimal amount, string tokenAddress)
+            Task.WaitAll(tasks.ToArray());
-        {
-            log.Debug($"({tokenAddress}) {amount} --> {account}");
-            if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
-
-            var function = new MintTokensFunction
-            {
-                Holder = account,
-                Amount = ToBig(amount)
-            };
-
-            var handler = web3.Eth.GetContractTransactionHandler<MintTokensFunction>();
-            openTasks.Add(handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function));
         }
 
         public decimal GetBalance(string tokenAddress, string account)
@@ -68,48 +49,54 @@ namespace NethereumWorkflow
             return ToDecimal(Time.Wait(handler.QueryAsync<BigInteger>(tokenAddress, function)));
         }
 
-        public void WaitForAllTransactions()
+        public bool IsSynced(string marketplaceAddress, string marketplaceAbi)
         {
-            var tasks = openTasks.ToArray();
+            try
-            openTasks.Clear();
+            {
-
+                return IsBlockNumberOK() && IsContractAvailable(marketplaceAddress, marketplaceAbi);
-            Task.WaitAll(tasks);
+            }
+            catch
+            {
+                return false;
+            }
         }
 
-        public void EnsureSynced(string marketplaceAddress, string marketplaceAbi)
+        private Task MintTokens(string account, decimal amount, string tokenAddress)
         {
-            WaitUntilSynced();
+            log.Debug($"({tokenAddress}) {amount} --> {account}");
-            WaitForContract(marketplaceAddress, marketplaceAbi);
+            if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
+
+            var function = new MintTokensFunction
+            {
+                Holder = account,
+                Amount = ToBig(amount)
+            };
+
+            var handler = web3.Eth.GetContractTransactionHandler<MintTokensFunction>();
+            return handler.SendRequestAndWaitForReceiptAsync(tokenAddress, function);
         }
 
-        private void WaitUntilSynced()
+        private bool IsBlockNumberOK()
         {
             log.Debug();
-            Time.WaitUntil(() =>
+            var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync());
-            {
+            var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
-                var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync());
+            var numberOfBlocks = ToDecimal(number);
-                var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
+            return !sync.IsSyncing && numberOfBlocks > 256;
-                var numberOfBlocks = ToDecimal(number);
-                return !sync.IsSyncing && numberOfBlocks > 256;
-
-            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
         }
 
-        private void WaitForContract(string marketplaceAddress, string marketplaceAbi)
+        private bool IsContractAvailable(string marketplaceAddress, string marketplaceAbi)
         {
             log.Debug();
-            Time.WaitUntil(() =>
+            try
             {
-                try
+                var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress);
-                {
+                return contract != null;
-                    var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress);
+            }
-                    return contract != null;
+            catch
-                }
+            {
-                catch
+                return false;
-                {
+            }
-                    return false;
-                }
-            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
         }
 
         private HexBigInteger ToHexBig(decimal amount)

Nethereum/NethereumInteractionCreator.cs

@@ -8,21 +8,19 @@ namespace NethereumWorkflow
         private readonly BaseLog log;
         private readonly string ip;
         private readonly int port;
-        private readonly string rootAccount;
         private readonly string privateKey;
 
-        public NethereumInteractionCreator(BaseLog log, string ip, int port, string rootAccount, string privateKey)
+        public NethereumInteractionCreator(BaseLog log, string ip, int port, string privateKey)
         {
             this.log = log;
             this.ip = ip;
             this.port = port;
-            this.rootAccount = rootAccount;
             this.privateKey = privateKey;
         }
 
         public NethereumInteraction CreateWorkflow()
         {
-            return new NethereumInteraction(log, CreateWeb3(), rootAccount);
+            return new NethereumInteraction(log, CreateWeb3());
         }
 
         private Web3 CreateWeb3()

Nethereum/NethereumWorkflow.csproj

@@ -1,16 +1,16 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 	<RootNamespace>NethereumWorkflow</RootNamespace>
 	<ImplicitUsings>enable</ImplicitUsings>
 	<Nullable>enable</Nullable>
-	  
+
   </PropertyGroup>
 	<ItemGroup>
 		<PackageReference Include="Nethereum.Web3" Version="4.14.0" />
 	</ItemGroup>
-	
+
   <ItemGroup>
     <ProjectReference Include="..\Logging\Logging.csproj" />
     <ProjectReference Include="..\Utils\Utils.csproj" />

Tests/BasicTests/DownloadTests.cs

@@ -36,4 +36,4 @@ namespace Tests.ParallelTests
             }
         }
     }
-}
+}

Tests/BasicTests/ExampleTests.cs

@@ -71,13 +71,13 @@ namespace Tests.BasicTests
                 requiredCollateral: 10.TestTokens(),
                 minRequiredNumberOfNodes: 1,
                 proofProbability: 5,
-                duration: TimeSpan.FromMinutes(2));
+                duration: TimeSpan.FromMinutes(1));
 
-            Time.Sleep(TimeSpan.FromMinutes(1));
+            Time.Sleep(TimeSpan.FromSeconds(10));
 
             seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
 
-            Time.Sleep(TimeSpan.FromMinutes(2));
+            Time.Sleep(TimeSpan.FromMinutes(1));
 
             seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
             buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");

Tests/BasicTests/NetworkIsolationTest.cs

@@ -0,0 +1,45 @@
+using DistTestCore;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.BasicTests
+{
+    // Warning!
+    // This is a test to check network-isolation in the test-infrastructure.
+    // It requires parallelism(2) or greater to run.
+    [TestFixture]
+    public class NetworkIsolationTest : DistTest
+    {
+        private IOnlineCodexNode? node = null;
+
+        [Test]
+        public void SetUpANodeAndWait()
+        {
+            node = SetupCodexNode();
+
+            Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5));
+        }
+
+        [Test]
+        public void ForeignNodeConnects()
+        {
+            var myNode = SetupCodexNode();
+
+            Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5));
+
+            try
+            {
+                myNode.ConnectToPeer(node!);
+            }
+            catch
+            {
+                // Good! This connection should be prohibited by the network isolation policy.
+                node = null;
+                return;
+            }
+
+            Assert.Fail("Connection could be established between two Codex nodes running in different namespaces. " +
+                "This may cause cross-test interference. Network isolation policy should be applied. Test infra failure.");
+        }
+    }
+}

Tests/BasicTests/PeerTests.cs

@@ -72,13 +72,13 @@ namespace Tests.BasicTests
 
         private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b)
         {
-            //var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
+            var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
             var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId));
 
-            //Log.Debug($"Looking for {b.id} in engine-peers [{enginePeers}]");
+            Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]");
-            Log.Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
+            Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
 
-            //Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
+            Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
             Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found.");
         }
     }

Tests/BasicTests/TwoClientTests.cs

@@ -7,15 +7,23 @@ namespace Tests.BasicTests
     [TestFixture]
     public class TwoClientTests : DistTest
     {
-        [Test]
+        [TestCase(1)]
-        public void TwoClientsOnePodTest()
+        [TestCase(2)]
+        [TestCase(3)]
+        [TestCase(4)]
+        [TestCase(5)]
+        [TestCase(6)]
+        [TestCase(7)]
+        [TestCase(8)]
+        [TestCase(9)]
+        public void TwoClientsOnePodTest(int size)
         {
             var group = SetupCodexNodes(2);
 
             var primary = group[0];
             var secondary = group[1];
 
-            PerformTwoClientTest(primary, secondary);
+            PerformTwoClientTest(primary, secondary, size.MB());
         }
 
         [Test]
@@ -38,10 +46,15 @@ namespace Tests.BasicTests
         }
 
         private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
+        {
+            PerformTwoClientTest(primary, secondary, 1.MB());
+        }
+
+        private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary, ByteSize size)
         {
             primary.ConnectToPeer(secondary);
 
-            var testFile = GenerateTestFile(1.MB());
+            var testFile = GenerateTestFile(size);
 
             var contentId = primary.UploadFile(testFile);
 

Tests/BasicTests/UploadTests.cs

@@ -42,4 +42,4 @@ namespace Tests.ParallelTests
             }
         }
     }
-}
+}

Tests/Parallelism.cs

@@ -0,0 +1,6 @@
+using NUnit.Framework;
+
+[assembly: LevelOfParallelism(1)]
+namespace Tests
+{
+}

Tests/Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Utils/Utils.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 	  <RootNamespace>Utils</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>