Merge branch 'feature/marketplace-contracts'

DistTestCore/AutoBootstrapDistTest.cs

@@ -0,0 +1,30 @@
+namespace DistTestCore
+{
+    public class AutoBootstrapDistTest : DistTest
+    {
+        private IOnlineCodexNode? bootstrapNode;
+
+        public override IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
+        {
+            throw new Exception("AutoBootstrapDistTest creates and attaches a single boostrap node for you. " +
+                "If you want to control the bootstrap node from your test, please use DistTest instead.");
+        }
+
+        public override ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        {
+            var codexSetup = new CodexSetup(numberOfNodes);
+            setup(codexSetup);
+            codexSetup.WithBootstrapNode(EnsureBootstapNode());
+            return BringOnline(codexSetup);
+        }
+
+        private IOnlineCodexNode EnsureBootstapNode()
+        {
+            if (bootstrapNode == null)
+            {
+                bootstrapNode = base.SetupCodexBootstrapNode(s => { });
+            }
+            return bootstrapNode;
+        }
+    }
+}

DistTestCore/BaseStarter.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using Logging;
 
 namespace DistTestCore
 {

DistTestCore/Codex/CodexAccess.cs

@@ -1,11 +1,15 @@
 using KubernetesWorkflow;
+using Logging;
 
 namespace DistTestCore.Codex
 {
     public class CodexAccess
     {
-        public CodexAccess(RunningContainer runningContainer)
+        private readonly BaseLog log;
+
+        public CodexAccess(BaseLog log, RunningContainer runningContainer)
         {
+            this.log = log;
             Container = runningContainer;
         }
 
@@ -31,16 +35,16 @@ namespace DistTestCore.Codex
             return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
         }
 
-        public CodexSalesRequestStorageResponse RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
+        public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
         {
-            return Http().HttpPostJson<CodexSalesRequestStorageRequest, CodexSalesRequestStorageResponse>($"storage/request/{contentId}", request);
+            return Http().HttpPostJson($"storage/request/{contentId}", request);
         }
 
         private Http Http()
         {
             var ip = Container.Pod.Cluster.IP;
             var port = Container.ServicePorts[0].Number;
-            return new Http(ip, port, baseUrl: "/api/codex/v1");
+            return new Http(log, ip, port, baseUrl: "/api/codex/v1");
         }
 
         public string ConnectToPeer(string peerId, string peerMultiAddress)
@@ -55,9 +59,31 @@ namespace DistTestCore.Codex
         public string[] addrs { get; set; } = new string[0];
         public string repo { get; set; } = string.Empty;
         public string spr { get; set; } = string.Empty;
+        public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
+        public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
         public CodexDebugVersionResponse codex { get; set; } = new();
     }
 
+    public class EnginePeerResponse
+    {
+        public string peerId { get; set; } = string.Empty;
+        public EnginePeerContextResponse context { get; set; } = new();
+    }
+
+    public class EnginePeerContextResponse
+    {
+        public int blocks { get; set; } = 0;
+        public int peerWants { get; set; } = 0;
+        public int exchanged { get; set; } = 0;
+        public string lastExchange { get; set; } = string.Empty;
+    }
+
+    public class SwitchPeerResponse
+    {
+        public string peerId { get; set; } = string.Empty;
+        public string key { get; set; } = string.Empty;
+    }
+
     public class CodexDebugVersionResponse
     {
         public string version { get; set; } = string.Empty;
@@ -91,9 +117,4 @@ namespace DistTestCore.Codex
         public uint? nodes { get; set; }
         public uint? tolerance { get; set;}
     }
-
-    public class CodexSalesRequestStorageResponse
-    {
-        public string purchaseId { get; set; } = string.Empty;
-    }
 }

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -5,7 +5,8 @@ namespace DistTestCore.Codex
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        public const string DockerImage = "thatbenbierens/nim-codex:sha-bf5512b";
+        //public const string DockerImage = "thatbenbierens/nim-codex:sha-9716635";
+        public const string DockerImage = "thatbenbierens/codexlocal:latest";
         public const string MetricsPortTag = "metrics_port";
 
         protected override string Image => DockerImage;
@@ -21,6 +22,11 @@ namespace DistTestCore.Codex
             var listenPort = AddInternalPort();
             AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
 
+            if (!string.IsNullOrEmpty(config.BootstrapSpr))
+            {
+                AddEnvVar("BOOTSTRAP_SPR", config.BootstrapSpr);
+            }
+
             if (config.LogLevel != null)
             {
                 AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
@@ -38,14 +44,15 @@ namespace DistTestCore.Codex
             if (config.MarketplaceConfig != null)
             {
                 var gethConfig = startupConfig.Get<GethStartResult>();
-                var companionNode = gethConfig.CompanionNodes[Index];
+                var companionNode = gethConfig.CompanionNode;
-                Additional(companionNode);
+                var companionNodeAccount = companionNode.Accounts[Index];
+                Additional(companionNodeAccount);
 
                 var ip = companionNode.RunningContainer.Pod.Ip;
-                var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.WsPortTag).Number;
+                var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number;
 
                 AddEnvVar("ETH_PROVIDER", $"ws://{ip}:{port}");
-                AddEnvVar("ETH_ACCOUNT", companionNode.Account);
+                AddEnvVar("ETH_ACCOUNT", companionNodeAccount.Account);
                 AddEnvVar("ETH_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
             }
         }

DistTestCore/Codex/CodexStartupConfig.cs

@@ -5,11 +5,12 @@ namespace DistTestCore.Codex
 {
     public class CodexStartupConfig
     {
+        public string? NameOverride { get; set; }
         public Location Location { get; set; }
         public CodexLogLevel? LogLevel { get; set; }
         public ByteSize? StorageQuota { get; set; }
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
-        public IOnlineCodexNode? BootstrapNode { get; set; }
+        public string? BootstrapSpr { get; set; }
     }
 }

DistTestCore/CodexNodeGroup.cs

@@ -64,7 +64,7 @@ namespace DistTestCore
 
         private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
         {
-            var access = new CodexAccess(c);
+            var access = new CodexAccess(lifecycle.Log, c);
             EnsureOnline(access);
             return factory.CreateOnlineCodexNode(access, this);
         }
@@ -75,6 +75,10 @@ namespace DistTestCore
             {
                 var debugInfo = access.GetDebugInfo();
                 if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
+
+                var nodePeerId = debugInfo.id;
+                var nodeName = access.Container.Name;
+                lifecycle.Log.AddStringReplace(nodePeerId, $"___{nodeName}___");
             }
             catch (Exception e)
             {

DistTestCore/CodexSetup.cs

@@ -6,6 +6,7 @@ namespace DistTestCore
 {
     public interface ICodexSetup
     {
+        ICodexSetup WithName(string name);
         ICodexSetup At(Location location);
         ICodexSetup WithLogLevel(CodexLogLevel level);
         ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
@@ -13,41 +14,21 @@ namespace DistTestCore
         ICodexSetup EnableMetrics();
         ICodexSetup EnableMarketplace(TestToken initialBalance);
         ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
-        ICodexNodeGroup BringOnline();
     }
     
     public class CodexSetup : CodexStartupConfig, ICodexSetup
     {
-        private readonly CodexStarter starter;
-
         public int NumberOfNodes { get; }
 
-        public CodexSetup(CodexStarter starter, int numberOfNodes)
+        public CodexSetup(int numberOfNodes)
         {
-            this.starter = starter;
             NumberOfNodes = numberOfNodes;
         }
 
-        public ICodexNodeGroup BringOnline()
+        public ICodexSetup WithName(string name)
         {
-            var group = starter.BringOnline(this);
+            NameOverride = name;
-            ConnectToBootstrapNode(group);
+            return this;
-            return group;
-        }
-
-        private void ConnectToBootstrapNode(ICodexNodeGroup group)
-        {
-            if (BootstrapNode == null) return;
-
-            // TODO:
-            // node.ConnectToPeer uses the '/api/codex/vi/connect/' endpoint to make the connection.
-            // This should be replaced by injecting the bootstrap node's SPR into the env-vars of the new node containers. (Easy!)
-            // However, NAT isn't figure out yet. So connecting with SPR doesn't (always?) work.
-            // So for now, ConnectToPeer
-            foreach (var node in group)
-            {
-                node.ConnectToPeer(BootstrapNode);
-            }
         }
 
         public ICodexSetup At(Location location)
@@ -58,7 +39,7 @@ namespace DistTestCore
 
         public ICodexSetup WithBootstrapNode(IOnlineCodexNode node)
         {
-            BootstrapNode = node;
+            BootstrapSpr = node.GetDebugInfo().spr;
             return this;
         }
 
@@ -100,7 +81,7 @@ namespace DistTestCore
         private IEnumerable<string> DescribeArgs()
         {
             if (LogLevel != null) yield return $"LogLevel={LogLevel}";
-            if (BootstrapNode != null) yield return $"BootstrapNode={BootstrapNode.GetName()}";
+            if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
             if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
         }
     }

DistTestCore/CodexStarter.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Codex;
+using DistTestCore.Marketplace;
 using KubernetesWorkflow;
 
 namespace DistTestCore
@@ -18,10 +19,7 @@ namespace DistTestCore
             LogStart($"Starting {codexSetup.Describe()}...");
             var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
 
-            var startupConfig = new StartupConfig();
+            var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
-            startupConfig.Add(codexSetup);
-            startupConfig.Add(gethStartResult);
-
             var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
 
             var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers);
@@ -56,7 +54,16 @@ namespace DistTestCore
             var workflow = CreateWorkflow();
             workflow.DownloadContainerLog(container, logHandler);
         }
-        
+
+        private StartupConfig CreateStartupConfig(GethStartResult gethStartResult, CodexSetup codexSetup)
+        {
+            var startupConfig = new StartupConfig();
+            startupConfig.NameOverride = codexSetup.NameOverride;
+            startupConfig.Add(codexSetup);
+            startupConfig.Add(gethStartResult);
+            return startupConfig;
+        }
+
         private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
         {
             var workflow = CreateWorkflow();

DistTestCore/Configuration.cs

@@ -21,7 +21,7 @@ namespace DistTestCore
 
         public Logging.LogConfig GetLogConfig()
         {
-            return new Logging.LogConfig("CodexTestLogs");
+            return new Logging.LogConfig("CodexTestLogs", debugEnabled: false);
         }
 
         public string GetFileManagerFolder()

DistTestCore/DistTest.cs

@@ -5,6 +5,7 @@ using DistTestCore.Metrics;
 using KubernetesWorkflow;
 using Logging;
 using NUnit.Framework;
+using System.Reflection;
 using Utils;
 
 namespace DistTestCore
@@ -13,22 +14,30 @@ namespace DistTestCore
     public abstract class DistTest
     {
         private readonly Configuration configuration = new Configuration();
+        private readonly Assembly[] testAssemblies;
         private FixtureLog fixtureLog = null!;
         private TestLifecycle lifecycle = null!;
         private DateTime testStart = DateTime.MinValue;
 
+        public DistTest()
+        {
+            var assemblies = AppDomain.CurrentDomain.GetAssemblies();
+            testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
+        }
+
         [OneTimeSetUp]
         public void GlobalSetup()
         {
             // Previous test run may have been interrupted.
             // Begin by cleaning everything up.
+            Timing.UseLongTimeouts = false;
             fixtureLog = new FixtureLog(configuration.GetLogConfig());
 
             try
             {
                 Stopwatch.Measure(fixtureLog, "Global setup", () =>
                 {
-                    var wc = new WorkflowCreator(configuration.GetK8sConfiguration());
+                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration());
                     wc.CreateWorkflow().DeleteAllResources();
                 });                
             }
@@ -48,6 +57,8 @@ namespace DistTestCore
         [SetUp]
         public void SetUpDistTest()
         {
+            Timing.UseLongTimeouts = ShouldUseLongTimeouts();
+
             if (GlobalTestFailure.HasFailed)
             {
                 Assert.Inconclusive("Skip test: Previous test failed during clean up.");
@@ -77,9 +88,67 @@ namespace DistTestCore
             return lifecycle.FileManager.GenerateTestFile(size);
         }
 
-        public ICodexSetup SetupCodexNodes(int numberOfNodes)
+        public IOnlineCodexNode SetupCodexBootstrapNode()
         {
-            return new CodexSetup(lifecycle.CodexStarter, numberOfNodes);
+            return SetupCodexBootstrapNode(s => { });
+        }
+
+        public virtual IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
+        {
+            return SetupCodexNode(s =>
+            {
+                setup(s);
+                s.WithName("Bootstrap");
+            });
+        }
+
+        public IOnlineCodexNode SetupCodexNode()
+        {
+            return SetupCodexNode(s => { });
+        }
+
+        public IOnlineCodexNode SetupCodexNode(Action<ICodexSetup> setup)
+        {
+            return SetupCodexNodes(1, setup)[0];
+        }
+
+        public ICodexNodeGroup SetupCodexNodes(int numberOfNodes)
+        {
+            return SetupCodexNodes(numberOfNodes, s => { });
+        }
+
+        public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        {
+            var codexSetup = new CodexSetup(numberOfNodes);
+
+            setup(codexSetup);
+
+            return BringOnline(codexSetup);
+        }
+
+        public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
+        {
+            return lifecycle.CodexStarter.BringOnline((CodexSetup)codexSetup);
+        }
+
+        protected BaseLog Log
+        {
+            get { return lifecycle.Log; }
+        }
+
+        private bool ShouldUseLongTimeouts()
+        {
+            // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
+            // But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
+            // if the attribute is present.
+            var currentTest = TestContext.CurrentContext.Test;
+            var className = currentTest.ClassName;
+            var methodName = currentTest.MethodName;
+
+            var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
+            var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
+
+            return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
         }
 
         private void CreateNewTestLifecycle()

DistTestCore/GethStarter.cs

@@ -22,37 +22,37 @@ namespace DistTestCore
             if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult();
 
             var marketplaceNetwork = marketplaceNetworkCache.Get();
-            var companionNodes = StartCompanionNodes(codexSetup, marketplaceNetwork);
+            var companionNode = StartCompanionNode(codexSetup, marketplaceNetwork);
 
             LogStart("Setting up initial balance...");
-            TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNodes);
+            TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNode);
             LogEnd($"Initial balance of {codexSetup.MarketplaceConfig.InitialTestTokens} set for {codexSetup.NumberOfNodes} nodes.");
 
-            return CreateGethStartResult(marketplaceNetwork, companionNodes);
+            return CreateGethStartResult(marketplaceNetwork, companionNode);
         }
 
-        private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo[] companionNodes)
+        private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
         {
             var interaction = marketplaceNetwork.StartInteraction(lifecycle.Log);
             var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
 
-            foreach (var node in companionNodes)
+            foreach (var account in companionNode.Accounts)
             {
-                interaction.TransferWeiTo(node.Account, marketplaceConfig.InitialEth.Wei);
+                interaction.TransferWeiTo(account.Account, marketplaceConfig.InitialEth.Wei);
-                interaction.MintTestTokens(node.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
+                interaction.MintTestTokens(account.Account, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
             }
 
             interaction.WaitForAllTransactions();
         }
 
-        private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo[] companionNodes)
+        private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
         {
-            return new GethStartResult(CreateMarketplaceAccessFactory(marketplaceNetwork), marketplaceNetwork, companionNodes);
+            return new GethStartResult(CreateMarketplaceAccessFactory(marketplaceNetwork), marketplaceNetwork, companionNode);
         }
 
         private GethStartResult CreateMarketplaceUnavailableResult()
         {
-            return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, Array.Empty<GethCompanionNodeInfo>());
+            return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, null!);
         }
 
         private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork)
@@ -60,9 +60,9 @@ namespace DistTestCore
             return new GethMarketplaceAccessFactory(lifecycle.Log, marketplaceNetwork);
         }
 
-        private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)
+        private GethCompanionNodeInfo StartCompanionNode(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)
         {
-            return companionNodeStarter.StartCompanionNodesFor(codexSetup, marketplaceNetwork.Bootstrap);
+            return companionNodeStarter.StartCompanionNodeFor(codexSetup, marketplaceNetwork);
         }
     }
 

DistTestCore/Http.cs

@@ -1,4 +1,5 @@
-using Newtonsoft.Json;
+using Logging;
+using Newtonsoft.Json;
 using NUnit.Framework;
 using System.Net.Http.Headers;
 using System.Net.Http.Json;
@@ -8,12 +9,14 @@ namespace DistTestCore
 {
     public class Http
     {
+        private readonly BaseLog log;
         private readonly string ip;
         private readonly int port;
         private readonly string baseUrl;
 
-        public Http(string ip, int port, string baseUrl)
+        public Http(BaseLog log, string ip, int port, string baseUrl)
         {
+            this.log = log;
             this.ip = ip;
             this.port = port;
             this.baseUrl = baseUrl;
@@ -28,8 +31,11 @@ namespace DistTestCore
             {
                 using var client = GetClient();
                 var url = GetUrl() + route;
+                Log(url, "");
                 var result = Time.Wait(client.GetAsync(url));
-                return Time.Wait(result.Content.ReadAsStringAsync());
+                var str = Time.Wait(result.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str; ;
             });
         }
 
@@ -41,16 +47,23 @@ namespace DistTestCore
 
         public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
         {
-            var json = Retry(() =>
+            var json = HttpPostJson(route, body);
+            return TryJsonDeserialize<TResponse>(json);
+        }
+
+        public string HttpPostJson<TRequest>(string route, TRequest body)
+        {
+            return Retry(() =>
             {
                 using var client = GetClient();
                 var url = GetUrl() + route;
                 using var content = JsonContent.Create(body);
+                Log(url, JsonConvert.SerializeObject(body));
                 var result = Time.Wait(client.PostAsync(url, content));
-                return Time.Wait(result.Content.ReadAsStringAsync());
+                var str= Time.Wait(result.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
             });
-
-            return TryJsonDeserialize<TResponse>(json);
         }
 
         public string HttpPostStream(string route, Stream stream)
@@ -59,12 +72,13 @@ namespace DistTestCore
             {
                 using var client = GetClient();
                 var url = GetUrl() + route;
-
+                Log(url, "~ STREAM ~");
                 var content = new StreamContent(stream);
                 content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
                 var response = Time.Wait(client.PostAsync(url, content));
-
+                var str =Time.Wait(response.Content.ReadAsStringAsync());
-                return Time.Wait(response.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
             });
         }
 
@@ -74,7 +88,7 @@ namespace DistTestCore
             {
                 var client = GetClient();
                 var url = GetUrl() + route;
-
+                Log(url, "~ STREAM ~");
                 return Time.Wait(client.GetStreamAsync(url));
             });
         }
@@ -84,6 +98,11 @@ namespace DistTestCore
             return $"http://{ip}:{port}{baseUrl}";
         }
 
+        private void Log(string url, string message)
+        {
+            log.Debug($"({url}) = '{message}'", 3);
+        }
+
         private static T Retry<T>(Func<T> operation)
         {
             var retryCounter = 0;

DistTestCore/Marketplace/CodexContractsContainerRecipe.cs

@@ -6,6 +6,7 @@ namespace DistTestCore.Marketplace
     {
         public const string DockerImage = "thatbenbierens/codex-contracts-deployment";
         public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json";
+        public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json";
 
         protected override string Image => DockerImage;
 

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -30,15 +30,16 @@ namespace DistTestCore.Marketplace
                 return logHandler.Found;
             });
 
-            var extractor = new ContainerInfoExtractor(workflow, container);
+            var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
             var marketplaceAddress = extractor.ExtractMarketplaceAddress();
+            var abi = extractor.ExtractMarketplaceAbi();
 
             var interaction = bootstrapNode.StartInteraction(lifecycle.Log);
             var tokenAddress = interaction.GetTokenAddress(marketplaceAddress);
 
             LogEnd("Contracts deployed.");
 
-            return new MarketplaceInfo(marketplaceAddress, tokenAddress);
+            return new MarketplaceInfo(marketplaceAddress, abi, tokenAddress);
         }
 
         private void WaitUntil(Func<bool> predicate)
@@ -57,13 +58,15 @@ namespace DistTestCore.Marketplace
 
     public class MarketplaceInfo
     {
-        public MarketplaceInfo(string address, string tokenAddress)
+        public MarketplaceInfo(string address, string abi, string tokenAddress)
         {
             Address = address;
+            Abi = abi;
             TokenAddress = tokenAddress;
         }
 
         public string Address { get; }
+        public string Abi { get; }
         public string TokenAddress { get; }
     }
 

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -1,23 +1,28 @@
 using KubernetesWorkflow;
+using Logging;
 using Newtonsoft.Json;
-using System.Text;
+using Newtonsoft.Json.Linq;
+using Utils;
 
 namespace DistTestCore.Marketplace
 {
     public class ContainerInfoExtractor
     {
+        private readonly BaseLog log;
         private readonly StartupWorkflow workflow;
         private readonly RunningContainer container;
 
-        public ContainerInfoExtractor(StartupWorkflow workflow, RunningContainer container)
+        public ContainerInfoExtractor(BaseLog log, StartupWorkflow workflow, RunningContainer container)
         {
+            this.log = log;
             this.workflow = workflow;
             this.container = container;
         }
 
-        public string ExtractAccount()
+        public string ExtractAccount(int? orderNumber)
         {
-            var account = Retry(FetchAccount);
+            log.Debug();
+            var account = Retry(() => FetchAccount(orderNumber));
             if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure.");
 
             return account;
@@ -25,15 +30,17 @@ namespace DistTestCore.Marketplace
 
         public string ExtractPubKey()
         {
+            log.Debug();
             var pubKey = Retry(FetchPubKey);
             if (string.IsNullOrEmpty(pubKey)) throw new InvalidOperationException("Unable to fetch enode from geth node. Test infra failure.");
 
             return pubKey;
         }
 
-        public string ExtractBootstrapPrivateKey()
+        public string ExtractPrivateKey(int? orderNumber)
         {
-            var privKey = Retry(FetchBootstrapPrivateKey);
+            log.Debug();
+            var privKey = Retry(() => FetchPrivateKey(orderNumber));
             if (string.IsNullOrEmpty(privKey)) throw new InvalidOperationException("Unable to fetch private key from geth node. Test infra failure.");
 
             return privKey;
@@ -41,20 +48,31 @@ namespace DistTestCore.Marketplace
 
         public string ExtractMarketplaceAddress()
         {
+            log.Debug();
             var marketplaceAddress = Retry(FetchMarketplaceAddress);
             if (string.IsNullOrEmpty(marketplaceAddress)) throw new InvalidOperationException("Unable to fetch marketplace account from codex-contracts node. Test infra failure.");
 
             return marketplaceAddress;
         }
 
+        public string ExtractMarketplaceAbi()
+        {
+            log.Debug();
+            var marketplaceAbi = Retry(FetchMarketplaceAbi);
+            if (string.IsNullOrEmpty(marketplaceAbi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
+
+            return marketplaceAbi;
+        }
+
         private string Retry(Func<string> fetch)
         {
-            var result = Catch(fetch);
+            var result = string.Empty;
-            if (string.IsNullOrEmpty(result))
+            Time.WaitUntil(() =>
             {
-                Thread.Sleep(TimeSpan.FromSeconds(5));
+                result = Catch(fetch);
-                result = fetch();
+                return !string.IsNullOrEmpty(result);
-            }
+            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
+
             return result;
         }
 
@@ -70,14 +88,14 @@ namespace DistTestCore.Marketplace
             }
         }
 
-        private string FetchAccount()
+        private string FetchAccount(int? orderNumber)
         {
-            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename);
+            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetAccountFilename(orderNumber));
         }
 
-        private string FetchBootstrapPrivateKey()
+        private string FetchPrivateKey(int? orderNumber)
         {
-            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.BootstrapPrivateKeyFilename);
+            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GetPrivateKeyFilename(orderNumber));
         }
 
         private string FetchMarketplaceAddress()
@@ -87,6 +105,15 @@ namespace DistTestCore.Marketplace
             return marketplace!.address;
         }
 
+        private string FetchMarketplaceAbi()
+        {
+            var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceArtifactFilename);
+
+            var artifact = JObject.Parse(json);
+            var abi = artifact["abi"];
+            return abi!.ToString(Formatting.None);
+        }
+
         private string FetchPubKey()
         {
             var enodeFinder = new PubKeyFinder();
@@ -97,7 +124,8 @@ namespace DistTestCore.Marketplace
 
     public class PubKeyFinder : LogHandler, ILogHandler
     {
-        private const string openTag = "self=\"enode://";
+        private const string openTag = "self=enode://";
+        private const string openTagQuote = "self=\"enode://";
         private string pubKey = string.Empty;
 
         public string GetPubKey()
@@ -109,13 +137,17 @@ namespace DistTestCore.Marketplace
         {
             if (line.Contains(openTag))
             {
-                ExtractPubKey(line);
+                ExtractPubKey(openTag, line);
+            }
+            else if (line.Contains(openTagQuote))
+            {
+                ExtractPubKey(openTagQuote, line);
             }
         }
 
-        private void ExtractPubKey(string line)
+        private void ExtractPubKey(string tag, string line)
         {
-            var openIndex = line.IndexOf(openTag) + openTag.Length;
+            var openIndex = line.IndexOf(tag) + tag.Length;
             var closeIndex = line.IndexOf("@");
 
             pubKey = line.Substring(

DistTestCore/Marketplace/GethBootstrapNodeInfo.cs

@@ -21,7 +21,7 @@ namespace DistTestCore.Marketplace
         public string PrivateKey { get; }
         public Port DiscoveryPort { get; }
 
-        public NethereumInteraction StartInteraction(TestLog log)
+        public NethereumInteraction StartInteraction(BaseLog log)
         {
             var ip = RunningContainers.RunningPod.Cluster.IP;
             var port = RunningContainers.Containers[0].ServicePorts[0].Number;

DistTestCore/Marketplace/GethBootstrapNodeStarter.cs

@@ -19,10 +19,10 @@ namespace DistTestCore.Marketplace
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
             var bootstrapContainer = containers.Containers[0];
 
-            var extractor = new ContainerInfoExtractor(workflow, bootstrapContainer);
+            var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer);
-            var account = extractor.ExtractAccount();
+            var account = extractor.ExtractAccount(null);
             var pubKey = extractor.ExtractPubKey();
-            var privateKey = extractor.ExtractBootstrapPrivateKey();
+            var privateKey = extractor.ExtractPrivateKey(null);
             var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
 
             LogEnd($"Geth bootstrap node started with account '{account}'");
@@ -33,7 +33,7 @@ namespace DistTestCore.Marketplace
         private StartupConfig CreateBootstrapStartupConfig()
         {
             var config = new StartupConfig();
-            config.Add(new GethStartupConfig(true, null!));
+            config.Add(new GethStartupConfig(true, null!, 0));
             return config;
         }
     }

DistTestCore/Marketplace/GethCompanionNodeInfo.cs

@@ -1,16 +1,41 @@
 using KubernetesWorkflow;
+using Logging;
+using NethereumWorkflow;
 
 namespace DistTestCore.Marketplace
 {
     public class GethCompanionNodeInfo
     {
-        public GethCompanionNodeInfo(RunningContainer runningContainer, string account)
+        public GethCompanionNodeInfo(RunningContainer runningContainer, GethCompanionAccount[] accounts)
         {
             RunningContainer = runningContainer;
-            Account = account;
+            Accounts = accounts;
         }
 
         public RunningContainer RunningContainer { get; }
+        public GethCompanionAccount[] Accounts { get; }
+        
+        public NethereumInteraction StartInteraction(BaseLog log, GethCompanionAccount account)
+        {
+            var ip = RunningContainer.Pod.Cluster.IP;
+            var port = RunningContainer.ServicePorts[0].Number;
+            var accountStr = account.Account;
+            var privateKey = account.PrivateKey;
+
+            var creator = new NethereumInteractionCreator(log, ip, port, accountStr, privateKey);
+            return creator.CreateWorkflow();
+        }
+    }
+
+    public class GethCompanionAccount
+    {
+        public GethCompanionAccount(string account, string privateKey)
+        {
+            Account = account;
+            PrivateKey = privateKey;
+        }
+
         public string Account { get; }
+        public string PrivateKey { get; }
     }
 }

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using Utils;
 
 namespace DistTestCore.Marketplace
 {
@@ -9,34 +10,68 @@ namespace DistTestCore.Marketplace
         {
         }
 
-        public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode)
+        public GethCompanionNodeInfo StartCompanionNodeFor(CodexSetup codexSetup, MarketplaceNetwork marketplace)
         {
-            LogStart($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes.");
+            LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes.");
 
-            var startupConfig = CreateCompanionNodeStartupConfig(bootstrapNode);
+            var startupConfig = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
 
             var workflow = workflowCreator.CreateWorkflow();
-            var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig);
+            var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
-            if (containers.Containers.Length != codexSetup.NumberOfNodes) throw new InvalidOperationException("Expected a Geth companion node to be created for each Codex node. Test infra failure.");
+            WaitForAccountCreation(codexSetup.NumberOfNodes);
+            if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
+            var container = containers.Containers[0];
 
-            var result = containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray();
+            var node = CreateCompanionInfo(workflow, container, codexSetup.NumberOfNodes);
+            EnsureCompanionNodeIsSynced(node, marketplace);
 
-            LogEnd($"Initialized {codexSetup.NumberOfNodes} companion nodes. Their accounts: [{string.Join(",", result.Select(c => c.Account))}]");
+            LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]");
-
+            return node;
-            return result;
         }
 
-        private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container)
+        private void WaitForAccountCreation(int numberOfNodes)
         {
-            var extractor = new ContainerInfoExtractor(workflow, container);
+            // We wait proportional to the number of account the node has to create. It takes a few seconds for each one to generate the keys and create the files
-            var account = extractor.ExtractAccount();
+            // we will be trying to read in 'ExtractAccount', later on in the start-up process.
-            return new GethCompanionNodeInfo(container, account);
+            Time.Sleep(TimeSpan.FromSeconds(4.5 * numberOfNodes));
         }
 
-        private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode)
+        private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container, int numberOfAccounts)
+        {
+            var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
+            var accounts = ExtractAccounts(extractor, numberOfAccounts).ToArray();
+            return new GethCompanionNodeInfo(container, accounts);
+        }
+
+        private IEnumerable<GethCompanionAccount> ExtractAccounts(ContainerInfoExtractor extractor, int numberOfAccounts)
+        {
+            for (int i = 0; i < numberOfAccounts; i++) yield return ExtractAccount(extractor, i + 1);
+        }
+
+        private GethCompanionAccount ExtractAccount(ContainerInfoExtractor extractor, int orderNumber)
+        {
+            var account = extractor.ExtractAccount(orderNumber);
+            var privKey = extractor.ExtractPrivateKey(orderNumber);
+            return new GethCompanionAccount(account, privKey);
+        }
+
+        private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
+        {
+            try
+            {
+                var interaction = node.StartInteraction(lifecycle.Log, node.Accounts.First());
+                interaction.EnsureSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
+            }
+            catch (Exception e)
+            {
+                throw new Exception("Geth companion node did not sync within timeout. Test infra failure.", e);
+            }
+        }
+
+        private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
         {
             var config = new StartupConfig();
-            config.Add(new GethStartupConfig(false, bootstrapNode));
+            config.Add(new GethStartupConfig(false, bootstrapNode, numberOfAccounts));
             return config;
         }
     }

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -6,10 +6,20 @@ namespace DistTestCore.Marketplace
     {
         public const string DockerImage = "thatbenbierens/geth-confenv:latest";
         public const string HttpPortTag = "http_port";
-        public const string WsPortTag = "ws_port";
         public const string DiscoveryPortTag = "disc_port";
-        public const string AccountFilename = "account_string.txt";
+        private const string defaultArgs = "--ipcdisable --syncmode full";
-        public const string BootstrapPrivateKeyFilename = "bootstrap_private.key";
+
+        public static string GetAccountFilename(int? orderNumber)
+        {
+            if (orderNumber == null) return "account_string.txt";
+            return $"account_string_{orderNumber.Value}.txt";
+        }
+
+        public static string GetPrivateKeyFilename(int? orderNumber)
+        {
+            if (orderNumber == null) return "private.key";
+            return $"private_{orderNumber.Value}.key";
+        }
 
         protected override string Image => DockerImage;
 
@@ -28,22 +38,33 @@ namespace DistTestCore.Marketplace
 
             if (config.IsBootstrapNode)
             {
-                AddEnvVar("IS_BOOTSTRAP", "1");
+                return CreateBootstapArgs(discovery);
-                var exposedPort = AddExposedPort(tag: HttpPortTag);
-                return $"--http.port {exposedPort.Number} --discovery.port {discovery.Number} --nodiscover";
             }
 
+            return CreateCompanionArgs(discovery, config);
+        }
+
+        private string CreateBootstapArgs(Port discovery)
+        {
+            AddEnvVar("IS_BOOTSTRAP", "1");
+            var exposedPort = AddExposedPort(tag: HttpPortTag);
+            return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}";
+        }
+
+        private string CreateCompanionArgs(Port discovery, GethStartupConfig config)
+        {
+            AddEnvVar("NUMBER_OF_ACCOUNTS", config.NumberOfCompanionAccounts.ToString());
+
             var port = AddInternalPort();
             var authRpc = AddInternalPort();
-            var httpPort = AddInternalPort(tag: HttpPortTag);
+            var httpPort = AddExposedPort(tag: HttpPortTag);
-            var wsPort = AddInternalPort(tag: WsPortTag);
 
             var bootPubKey = config.BootstrapNode.PubKey;
             var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip;
             var bootPort = config.BootstrapNode.DiscoveryPort.Number;
-            var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort}";
+            var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}";
 
-            return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {wsPort.Number} --nodiscover {bootstrapArg}";
+            return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}";
         }
     }
 }

DistTestCore/Marketplace/GethStartResult.cs

@@ -2,15 +2,15 @@
 {
     public class GethStartResult
     {
-        public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo[] companionNodes)
+        public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
         {
             MarketplaceAccessFactory = marketplaceAccessFactory;
             MarketplaceNetwork = marketplaceNetwork;
-            CompanionNodes = companionNodes;
+            CompanionNode = companionNode;
         }
 
         public IMarketplaceAccessFactory MarketplaceAccessFactory { get; }
         public MarketplaceNetwork MarketplaceNetwork { get; }
-        public GethCompanionNodeInfo[] CompanionNodes { get; }
+        public GethCompanionNodeInfo CompanionNode { get; }
     }
 }

DistTestCore/Marketplace/GethStartupConfig.cs

@@ -2,13 +2,15 @@
 {
     public class GethStartupConfig
     {
-        public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode)
+        public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int numberOfCompanionAccounts)
         {
             IsBootstrapNode = isBootstrapNode;
             BootstrapNode = bootstrapNode;
+            NumberOfCompanionAccounts = numberOfCompanionAccounts;
         }
 
         public bool IsBootstrapNode { get; }
         public GethBootstrapNodeInfo BootstrapNode { get; }
+        public int NumberOfCompanionAccounts { get; }
     }
 }

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -19,14 +19,14 @@ namespace DistTestCore.Marketplace
     {
         private readonly TestLog log;
         private readonly MarketplaceNetwork marketplaceNetwork;
-        private readonly GethCompanionNodeInfo companionNode;
+        private readonly GethCompanionAccount account;
         private readonly CodexAccess codexAccess;
 
-        public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode, CodexAccess codexAccess)
+        public MarketplaceAccess(TestLog log, MarketplaceNetwork marketplaceNetwork, GethCompanionAccount account, CodexAccess codexAccess)
         {
             this.log = log;
             this.marketplaceNetwork = marketplaceNetwork;
-            this.companionNode = companionNode;
+            this.account = account;
             this.codexAccess = codexAccess;
         }
 
@@ -52,9 +52,14 @@ namespace DistTestCore.Marketplace
 
             var response = codexAccess.RequestStorage(request, contentId.Id);
 
-            Log($"Storage requested successfully. PurchaseId: {response.purchaseId}");
+            if (response == "Purchasing not available")
+            {
+                throw new InvalidOperationException(response);
+            }
 
-            return response.purchaseId;
+            Log($"Storage requested successfully. PurchaseId: {response}");
+
+            return response;
         }
 
         public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration)
@@ -99,18 +104,17 @@ namespace DistTestCore.Marketplace
         public TestToken GetBalance()
         {
             var interaction = marketplaceNetwork.StartInteraction(log);
-            var account = companionNode.Account;
+            var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account.Account);
-            var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account);
             var balance = new TestToken(amount);
 
-            Log($"Balance of {account} is {balance}.");
+            Log($"Balance of {account.Account} is {balance}.");
 
             return balance;
         }
 
         private void Log(string msg)
         {
-            log.Log($"{codexAccess.Container.GetName()} {msg}");
+            log.Log($"{codexAccess.Container.Name} {msg}");
         }
     }
 

DistTestCore/Marketplace/MarketplaceAccessFactory.cs

@@ -33,10 +33,10 @@ namespace DistTestCore.Marketplace
             return new MarketplaceAccess(log, marketplaceNetwork, companionNode, access);
         }
 
-        private GethCompanionNodeInfo GetGethCompanionNode(CodexAccess access)
+        private GethCompanionAccount GetGethCompanionNode(CodexAccess access)
         {
-            var node = access.Container.Recipe.Additionals.Single(a => a is GethCompanionNodeInfo);
+            var account = access.Container.Recipe.Additionals.Single(a => a is GethCompanionAccount);
-            return (GethCompanionNodeInfo)node;
+            return (GethCompanionAccount)account;
         }
     }
 }

DistTestCore/Marketplace/MarketplaceNetwork.cs

@@ -14,7 +14,7 @@ namespace DistTestCore.Marketplace
         public GethBootstrapNodeInfo Bootstrap { get; }
         public MarketplaceInfo Marketplace { get; }
 
-        public NethereumInteraction StartInteraction(TestLog log)
+        public NethereumInteraction StartInteraction(BaseLog log)
         {
             return Bootstrap.StartInteraction(log);
         }

DistTestCore/Metrics/MetricsAccess.cs

@@ -29,7 +29,7 @@ namespace DistTestCore.Metrics
             var metricSet = GetMetricWithTimeout(metricName);
             var metricValue = metricSet.Values[0].Value;
 
-            log.Log($"{node.GetName()} metric '{metricName}' = {metricValue}");
+            log.Log($"{node.Name} metric '{metricName}' = {metricValue}");
 
             Assert.That(metricValue, constraint, message);
         }

DistTestCore/Metrics/MetricsAccessFactory.cs

@@ -28,7 +28,7 @@ namespace DistTestCore.Metrics
 
         public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
         {
-            var query = new MetricsQuery(prometheusContainer);
+            var query = new MetricsQuery(lifecycle.Log, prometheusContainer);
             return new MetricsAccess(lifecycle.Log, query, codexContainer);
         }
     }

DistTestCore/Metrics/MetricsQuery.cs

@@ -1,5 +1,6 @@
 using DistTestCore.Codex;
 using KubernetesWorkflow;
+using Logging;
 using System.Globalization;
 
 namespace DistTestCore.Metrics
@@ -8,11 +9,12 @@ namespace DistTestCore.Metrics
     {
         private readonly Http http;
 
-        public MetricsQuery(RunningContainers runningContainers)
+        public MetricsQuery(BaseLog log, RunningContainers runningContainers)
         {
             RunningContainers = runningContainers;
 
             http = new Http(
+                log,
                 runningContainers.RunningPod.Cluster.IP,
                 runningContainers.Containers[0].ServicePorts[0].Number,
                 "api/v1");

DistTestCore/OnlineCodexNode.cs

@@ -16,6 +16,7 @@ namespace DistTestCore
         ICodexNodeLog DownloadLog();
         IMetricsAccess Metrics { get; }
         IMarketplaceAccess Marketplace { get; }
+        ICodexSetup BringOffline();
     }
 
     public class OnlineCodexNode : IOnlineCodexNode
@@ -23,7 +24,6 @@ namespace DistTestCore
         private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
         private const string UploadFailedMessage = "Unable to store block";
         private readonly TestLifecycle lifecycle;
-        private CodexDebugResponse? debugInfo;
 
         public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess)
         {
@@ -41,14 +41,12 @@ namespace DistTestCore
 
         public string GetName()
         {
-            return CodexAccess.Container.GetName();
+            return CodexAccess.Container.Name;
         }
 
         public CodexDebugResponse GetDebugInfo()
         {
-            if (debugInfo != null) return debugInfo;
+            var debugInfo = CodexAccess.GetDebugInfo();
-
-            debugInfo = CodexAccess.GetDebugInfo();
             Log($"Got DebugInfo with id: '{debugInfo.id}'.");
             return debugInfo;
         }
@@ -92,6 +90,11 @@ namespace DistTestCore
             return lifecycle.DownloadLog(this);
         }
 
+        public ICodexSetup BringOffline()
+        {
+            return Group.BringOffline();
+        }
+
         private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
         {
             var multiAddress = peerInfo.addrs.First();

DistTestCore/Stopwatch.cs

@@ -1,36 +0,0 @@
-using Logging;
-using Utils;
-
-namespace DistTestCore
-{
-    public class Stopwatch
-    {
-        private readonly DateTime start = DateTime.UtcNow;
-        private readonly BaseLog log;
-        private readonly string name;
-
-        public Stopwatch(BaseLog log, string name)
-        {
-            this.log = log;
-            this.name = name;
-        }
-
-        public static void Measure(BaseLog log, string name, Action action)
-        {
-            var sw = Begin(log, name);
-            action();
-            sw.End();
-        }
-
-        public static Stopwatch Begin(BaseLog log, string name)
-        {
-            return new Stopwatch(log, name);
-        }
-
-        public void End(string msg = "")
-        {
-            var duration = DateTime.UtcNow - start;
-            log.Log($"{name} {msg} ({Time.FormatDuration(duration)})");
-        }
-    }
-}

DistTestCore/TestLifecycle.cs

@@ -11,7 +11,7 @@ namespace DistTestCore
         public TestLifecycle(TestLog log, Configuration configuration)
         {
             Log = log;
-            workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration());
+            workflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration());
 
             FileManager = new FileManager(Log, configuration);
             CodexStarter = new CodexStarter(this, workflowCreator);

DistTestCore/Timing.cs

@@ -6,15 +6,12 @@ namespace DistTestCore
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
     public class UseLongTimeoutsAttribute : PropertyAttribute
     {
-        public UseLongTimeoutsAttribute()
-            : base(Timing.UseLongTimeoutsKey)
-        {
-        }
     }
 
     public static class Timing
     {
-        public const string UseLongTimeoutsKey = "UseLongTimeouts";
+        public static bool UseLongTimeouts { get; set; }
+
 
         public static TimeSpan HttpCallTimeout()
         {
@@ -48,8 +45,7 @@ namespace DistTestCore
 
         private static ITimeSet GetTimes()
         {
-            var testProperties = TestContext.CurrentContext.Test.Properties;
+            if (UseLongTimeouts) return new LongTimeSet();
-            if (testProperties.ContainsKey(UseLongTimeoutsKey)) return new LongTimeSet();
             return new DefaultTimeSet();
         }
     }

DistTestCore/Tokens.cs

@@ -1,6 +1,6 @@
 namespace DistTestCore
 {
-    public class Ether
+    public class Ether : IComparable<Ether>
     {
         public Ether(decimal wei)
         {
@@ -9,6 +9,11 @@
 
         public decimal Wei { get; }
 
+        public int CompareTo(Ether? other)
+        {
+            return Wei.CompareTo(other!.Wei);
+        }
+
         public override bool Equals(object? obj)
         {
             return obj is Ether ether && Wei == ether.Wei;
@@ -25,7 +30,7 @@
         }
     }
 
-    public class TestToken
+    public class TestToken : IComparable<TestToken>
     {
         public TestToken(decimal amount)
         {
@@ -34,6 +39,11 @@
 
         public decimal Amount { get; }
 
+        public int CompareTo(TestToken? other)
+        {
+            return Amount.CompareTo(other!.Amount);
+        }
+
         public override bool Equals(object? obj)
         {
             return obj is TestToken token && Amount == token.Amount;

KubernetesWorkflow/K8sController.cs

@@ -1,18 +1,21 @@
 using k8s;
 using k8s.Models;
+using Logging;
 using Utils;
 
 namespace KubernetesWorkflow
 {
     public class K8sController
     {
+        private readonly BaseLog log;
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownPods;
         private readonly WorkflowNumberSource workflowNumberSource;
         private readonly Kubernetes client;
 
-        public K8sController(K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource)
+        public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource)
         {
+            this.log = log;
             this.cluster = cluster;
             this.knownPods = knownPods;
             this.workflowNumberSource = workflowNumberSource;
@@ -27,6 +30,7 @@ namespace KubernetesWorkflow
         
         public RunningPod BringOnline(ContainerRecipe[] containerRecipes, Location location)
         {
+            log.Debug();
             EnsureTestNamespace();
 
             var deploymentName = CreateDeployment(containerRecipes, location);
@@ -38,6 +42,7 @@ namespace KubernetesWorkflow
 
         public void Stop(RunningPod pod)
         {
+            log.Debug();
             if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName);
             DeleteDeployment(pod.DeploymentName);
             WaitUntilDeploymentOffline(pod.DeploymentName);
@@ -46,12 +51,14 @@ namespace KubernetesWorkflow
 
         public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
         {
+            log.Debug();
             using var stream = client.ReadNamespacedPodLog(pod.Name, K8sNamespace, recipe.Name);
             logHandler.Log(stream);
         }
 
         public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
         {
+            log.Debug($"{containerName}: {command} ({string.Join(",", args)})");
             var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
             runner.Run();
             return runner.GetStdOut();
@@ -59,6 +66,7 @@ namespace KubernetesWorkflow
 
         public void DeleteAllResources()
         {
+            log.Debug();
             DeleteNamespace();
 
             WaitUntilNamespaceDeleted();
@@ -346,7 +354,15 @@ namespace KubernetesWorkflow
 
         private void WaitUntil(Func<bool> predicate)
         {
-            Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.WaitForK8sServiceDelay());
+            var sw = Stopwatch.Begin(log, true);
+            try
+            {
+                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.WaitForK8sServiceDelay());
+            }
+            finally
+            {
+                sw.End("", 1);
+            }
         }
 
         #endregion

KubernetesWorkflow/KubernetesWorkflow.csproj

@@ -12,6 +12,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
     <ProjectReference Include="..\Utils\Utils.csproj" />
   </ItemGroup>
 

KubernetesWorkflow/RunningContainers.cs

@@ -15,26 +15,35 @@
 
         public string Describe()
         {
-            return string.Join(",", Containers.Select(c => c.GetName()));
+            return string.Join(",", Containers.Select(c => c.Name));
         }
     }
 
     public class RunningContainer
     {
-        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts)
+        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, StartupConfig startupConfig)
         {
             Pod = pod;
             Recipe = recipe;
             ServicePorts = servicePorts;
+            Name = GetContainerName(recipe, startupConfig);
         }
 
-        public string GetName()
+        public string Name { get; }
-        {
-            return $"<{Recipe.Name}>";
-        }
-
         public RunningPod Pod { get; }
         public ContainerRecipe Recipe { get; }
         public Port[] ServicePorts { get; }
+
+        private string GetContainerName(ContainerRecipe recipe, StartupConfig startupConfig)
+        {
+            if (!string.IsNullOrEmpty(startupConfig.NameOverride))
+            {
+                return $"<{startupConfig.NameOverride}{recipe.Number}>";
+            }
+            else
+            {
+                return $"<{recipe.Name}>";
+            }
+        }
     }
 }

KubernetesWorkflow/StartupConfig.cs

@@ -4,6 +4,8 @@
     {
         private readonly List<object> configs = new List<object>();
 
+        public string? NameOverride { get; set; }
+
         public void Add(object config)
         {
             configs.Add(config);

KubernetesWorkflow/StartupWorkflow.cs

@@ -1,16 +1,18 @@
-using System.IO;
+using Logging;
 
 namespace KubernetesWorkflow
 {
     public class StartupWorkflow
     {
+        private readonly BaseLog log;
         private readonly WorkflowNumberSource numberSource;
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownK8SPods;
         private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
 
-        internal StartupWorkflow(WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods)
+        internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods)
         {
+            this.log = log;
             this.numberSource = numberSource;
             this.cluster = cluster;
             this.knownK8SPods = knownK8SPods;
@@ -24,7 +26,7 @@ namespace KubernetesWorkflow
 
                 var runningPod = controller.BringOnline(recipes, location);
 
-                return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes));
+                return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig));
             });
         }
 
@@ -60,13 +62,15 @@ namespace KubernetesWorkflow
             });
         }
 
-        private static RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes)
+        private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig)
         {
-            return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r))).ToArray();
+            log.Debug();
+            return recipes.Select(r => new RunningContainer(runningPod, r, runningPod.GetServicePortsForContainerRecipe(r), startupConfig)).ToArray();
         }
 
         private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
+            log.Debug();
             var result = new List<ContainerRecipe>();
             for (var i = 0; i < numberOfContainers; i++)
             {
@@ -78,14 +82,14 @@ namespace KubernetesWorkflow
 
         private void K8s(Action<K8sController> action)
         {
-            var controller = new K8sController(cluster, knownK8SPods, numberSource);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
             action(controller);
             controller.Dispose();
         }
 
         private T K8s<T>(Func<K8sController, T> action)
         {
-            var controller = new K8sController(cluster, knownK8SPods, numberSource);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource);
             var result = action(controller);
             controller.Dispose();
             return result;

KubernetesWorkflow/WorkflowCreator.cs

@@ -1,4 +1,5 @@
-using Utils;
+using Logging;
+using Utils;
 
 namespace KubernetesWorkflow
 {
@@ -9,10 +10,12 @@ namespace KubernetesWorkflow
         private readonly NumberSource containerNumberSource = new NumberSource(0);
         private readonly KnownK8sPods knownPods = new KnownK8sPods();
         private readonly K8sCluster cluster;
+        private readonly BaseLog log;
 
-        public WorkflowCreator(Configuration configuration)
+        public WorkflowCreator(BaseLog log, Configuration configuration)
         {
             cluster = new K8sCluster(configuration);
+            this.log = log;
         }
 
         public StartupWorkflow CreateWorkflow()
@@ -21,7 +24,7 @@ namespace KubernetesWorkflow
                                                                     servicePortNumberSource,
                                                                     containerNumberSource);
 
-            return new StartupWorkflow(workflowNumberSource, cluster, knownPods);
+            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods);
         }
     }
 }

Logging/BaseLog.cs

@@ -1,10 +1,19 @@
-namespace Logging
+using Utils;
+
+namespace Logging
 {
     public abstract class BaseLog
     {
+        private readonly bool debug;
+        private readonly List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
         private bool hasFailed;
         private LogFile? logFile;
-        
+
+        protected BaseLog(bool debug)
+        {
+            this.debug = debug;
+        }
+
         protected abstract LogFile CreateLogFile();
 
         protected LogFile LogFile 
@@ -18,7 +27,17 @@
 
         public void Log(string message)
         {
-            LogFile.Write(message);
+            LogFile.Write(ApplyReplacements(message));
+        }
+
+        public void Debug(string message = "", int skipFrames = 0)
+        {
+            if (debug)
+            {
+                var callerName = DebugStack.GetCallerName(skipFrames);
+                // We don't use Log because in the debug output we should not have any replacements.
+                LogFile.Write($"(debug)({callerName}) {message}");
+            }
         }
 
         public void Error(string message)
@@ -32,5 +51,38 @@
             hasFailed = true;
             LogFile.ConcatToFilename("_FAILED");
         }
+
+        public void AddStringReplace(string from, string to)
+        {
+            replacements.Add(new BaseLogStringReplacement(from, to));
+        }
+
+        private string ApplyReplacements(string str)
+        {
+            foreach (var replacement in replacements)
+            {
+                str = replacement.Apply(str);
+            }
+            return str;
+        }
+    }
+
+    public class BaseLogStringReplacement
+    {
+        private readonly string from;
+        private readonly string to;
+
+        public BaseLogStringReplacement(string from, string to)
+        {
+            this.from = from;
+            this.to = to;
+
+            if (string.IsNullOrEmpty(from) || string.IsNullOrEmpty(to) || from == to) throw new ArgumentException();
+        }
+
+        public string Apply(string msg)
+        {
+            return msg.Replace(from, to);
+        }
     }
 }

Logging/FixtureLog.cs

@@ -6,18 +6,21 @@ namespace Logging
     {
         private readonly DateTime start;
         private readonly string fullName;
+        private readonly LogConfig config;
 
         public FixtureLog(LogConfig config)
+            : base(config.DebugEnabled)
         {
             start = DateTime.UtcNow;
             var folder = DetermineFolder(config);
             var fixtureName = GetFixtureName();
             fullName = Path.Combine(folder, fixtureName);
+            this.config = config;
         }
 
         public TestLog CreateTestLog()
         {
-            return new TestLog(fullName);
+            return new TestLog(fullName, config.DebugEnabled);
         }
 
         protected override LogFile CreateLogFile()

Logging/LogConfig.cs

@@ -2,11 +2,13 @@
 {
     public class LogConfig
     {
-        public LogConfig(string logRoot)
+        public LogConfig(string logRoot, bool debugEnabled)
         {
             LogRoot = logRoot;
+            DebugEnabled = debugEnabled;
         }
         
         public string LogRoot { get; }
+        public bool DebugEnabled { get; }
     }
 }

Logging/Stopwatch.cs

@@ -0,0 +1,61 @@
+using Utils;
+
+namespace Logging
+{
+    public class Stopwatch
+    {
+        private readonly DateTime start = DateTime.UtcNow;
+        private readonly BaseLog log;
+        private readonly string name;
+        private readonly bool debug;
+
+        private Stopwatch(BaseLog log, string name, bool debug)
+        {
+            this.log = log;
+            this.name = name;
+            this.debug = debug;
+        }
+
+        public static void Measure(BaseLog log, string name, Action action, bool debug = false)
+        {
+            var sw = Begin(log, name, debug);
+            action();
+            sw.End();
+        }
+
+        public static Stopwatch Begin(BaseLog log)
+        {
+            return Begin(log, "");
+        }
+
+        public static Stopwatch Begin(BaseLog log, string name)
+        {
+            return Begin(log, name, false);
+        }
+
+        public static Stopwatch Begin(BaseLog log, bool debug)
+        {
+            return Begin(log, "", debug);
+        }
+
+        public static Stopwatch Begin(BaseLog log, string name, bool debug)
+        {
+            return new Stopwatch(log, name, debug);
+        }
+
+        public void End(string msg = "", int skipFrames = 0)
+        {
+            var duration = DateTime.UtcNow - start;
+            var entry = $"{name} {msg} ({Time.FormatDuration(duration)})";
+
+            if (debug)
+            {
+                log.Debug(entry, 1 + skipFrames);
+            }
+            else
+            {
+                log.Log(entry);
+            }
+        }
+    }
+}

Logging/TestLog.cs

@@ -9,7 +9,8 @@ namespace Logging
         private readonly string methodName;
         private readonly string fullName;
 
-        public TestLog(string folder)
+        public TestLog(string folder, bool debug)
+            : base(debug)
         {
             methodName = GetMethodName();
             fullName = Path.Combine(folder, methodName);

LongTests/BasicTests/LargeFileTests.cs

@@ -10,10 +10,9 @@ namespace TestsLong.BasicTests
         [Test, UseLongTimeouts]
         public void OneClientLargeFileTest()
         {
-            var primary = SetupCodexNodes(1)
+            var primary = SetupCodexNode(s => s
                                 .WithLogLevel(CodexLogLevel.Warn)
-                                .WithStorageQuota(20.GB())
+                                .WithStorageQuota(20.GB()));
-                                .BringOnline()[0];
 
             var testFile = GenerateTestFile(10.GB());
 

LongTests/BasicTests/TestInfraTests.cs

@@ -9,9 +9,7 @@ namespace TestsLong.BasicTests
         [Test, UseLongTimeouts]
         public void TestInfraShouldHave1000AddressSpacesPerPod()
         {
-            var group = SetupCodexNodes(1000)
+            var group = SetupCodexNodes(1000, s => s.EnableMetrics()); // Increases use of port address space per node.
-                            .EnableMetrics() // Increases use of port address space per node.
-                            .BringOnline();
 
             var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray();
 
@@ -24,7 +22,7 @@ namespace TestsLong.BasicTests
         {
             for (var i = 0; i < 20; i++)
             {
-                var n = SetupCodexNodes(1).BringOnline()[0];
+                var n = SetupCodexNode();
 
                 Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().id));
             }
@@ -33,10 +31,9 @@ namespace TestsLong.BasicTests
         [Test, UseLongTimeouts]
         public void DownloadConsistencyTest()
         {
-            var primary = SetupCodexNodes(1)
+            var primary = SetupCodexNode(s => s
                             .WithLogLevel(CodexLogLevel.Trace)
-                            .WithStorageQuota(2.MB())
+                            .WithStorageQuota(2.MB()));
-                            .BringOnline()[0];
 
             var testFile = GenerateTestFile(1.MB());
 

Nethereum/NethereumInteraction.cs

@@ -11,11 +11,11 @@ namespace NethereumWorkflow
     public class NethereumInteraction
     {
         private readonly List<Task> openTasks = new List<Task>();
-        private readonly TestLog log;
+        private readonly BaseLog log;
         private readonly Web3 web3;
         private readonly string rootAccount;
 
-        internal NethereumInteraction(TestLog log, Web3 web3, string rootAccount)
+        internal NethereumInteraction(BaseLog log, Web3 web3, string rootAccount)
         {
             this.log = log;
             this.web3 = web3;
@@ -24,6 +24,7 @@ namespace NethereumWorkflow
 
         public string GetTokenAddress(string marketplaceAddress)
         {
+            log.Debug(marketplaceAddress);
             var function = new GetTokenFunction();
 
             var handler = web3.Eth.GetContractQueryHandler<GetTokenFunction>();
@@ -32,6 +33,7 @@ namespace NethereumWorkflow
 
         public void TransferWeiTo(string account, decimal amount)
         {
+            log.Debug($"{amount} --> {account}");
             if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for AddToBalance");
 
             var value = ToHexBig(amount);
@@ -41,6 +43,7 @@ namespace NethereumWorkflow
 
         public void MintTestTokens(string account, decimal amount, string tokenAddress)
         {
+            log.Debug($"({tokenAddress}) {amount} --> {account}");
             if (amount < 1 || string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
 
             var function = new MintTokensFunction
@@ -55,6 +58,7 @@ namespace NethereumWorkflow
 
         public decimal GetBalance(string tokenAddress, string account)
         {
+            log.Debug($"({tokenAddress}) {account}");
             var function = new GetTokenBalanceFunction
             {
                 Owner = account
@@ -72,6 +76,42 @@ namespace NethereumWorkflow
             Task.WaitAll(tasks);
         }
 
+        public void EnsureSynced(string marketplaceAddress, string marketplaceAbi)
+        {
+            WaitUntilSynced();
+            WaitForContract(marketplaceAddress, marketplaceAbi);
+        }
+
+        private void WaitUntilSynced()
+        {
+            log.Debug();
+            Time.WaitUntil(() =>
+            {
+                var sync = Time.Wait(web3.Eth.Syncing.SendRequestAsync());
+                var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
+                var numberOfBlocks = ToDecimal(number);
+                return !sync.IsSyncing && numberOfBlocks > 256;
+
+            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
+        }
+
+        private void WaitForContract(string marketplaceAddress, string marketplaceAbi)
+        {
+            log.Debug();
+            Time.WaitUntil(() =>
+            {
+                try
+                {
+                    var contract = web3.Eth.GetContract(marketplaceAbi, marketplaceAddress);
+                    return contract != null;
+                }
+                catch
+                {
+                    return false;
+                }
+            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
+        }
+
         private HexBigInteger ToHexBig(decimal amount)
         {
             var bigint = ToBig(amount);
@@ -84,6 +124,11 @@ namespace NethereumWorkflow
             return new BigInteger(amount);
         }
 
+        private decimal ToDecimal(HexBigInteger hexBigInteger)
+        {
+            return ToDecimal(hexBigInteger.Value);
+        }
+
         private decimal ToDecimal(BigInteger bigInteger)
         {
             return (decimal)bigInteger;
@@ -106,7 +151,7 @@ namespace NethereumWorkflow
     }
 
     [Function("balanceOf", "uint256")]
-    public class GetTokenBalanceFunction :FunctionMessage
+    public class GetTokenBalanceFunction : FunctionMessage
     {
         [Parameter("address", "owner", 1)]
         public string Owner { get; set; }

Nethereum/NethereumInteractionCreator.cs

@@ -5,13 +5,13 @@ namespace NethereumWorkflow
 {
     public class NethereumInteractionCreator
     {
-        private readonly TestLog log;
+        private readonly BaseLog log;
         private readonly string ip;
         private readonly int port;
         private readonly string rootAccount;
         private readonly string privateKey;
 
-        public NethereumInteractionCreator(TestLog log, string ip, int port, string rootAccount, string privateKey)
+        public NethereumInteractionCreator(BaseLog log, string ip, int port, string rootAccount, string privateKey)
         {
             this.log = log;
             this.ip = ip;

Tests/BasicTests/DownloadTests.cs

@@ -1,38 +1,26 @@
 using DistTestCore;
-using KubernetesWorkflow;
 using NUnit.Framework;
+
 namespace Tests.ParallelTests
 {
     [TestFixture]
     public class DownloadTests : DistTest
     {
-        [Test]
+        [TestCase(3, 500)]
-        public void ThreeNodeDownloads()
+        [TestCase(5, 100)]
+        [TestCase(10, 256)]
+        [UseLongTimeouts]
+        public void ParallelDownload(int numberOfNodes, int filesizeMb)
         {
-            ParallelDownload(3, 5000.MB());
+            var group = SetupCodexNodes(numberOfNodes);
-        }
+            var host = SetupCodexNode();
-        [Test]
-        public void FiveNodeDownloads()
-        {
-            ParallelDownload(5, 1000.MB());
-        }
-        [Test]
-        public void TenNodeDownloads()
-        {
-            ParallelDownload(10, 256.MB());
-        }
-
-        void ParallelDownload(int numberOfNodes, ByteSize filesize)
-        {
-            var group = SetupCodexNodes(numberOfNodes).BringOnline();
-            var host = SetupCodexNodes(1).BringOnline()[0];
 
             foreach (var node in group)
             {
                 host.ConnectToPeer(node);
             }
 
-            var testFile = GenerateTestFile(filesize);
+            var testFile = GenerateTestFile(filesizeMb.MB());
             var contentId = host.UploadFile(testFile);
             var list = new List<Task<TestFile?>>();
             

Tests/BasicTests/ExampleTests.cs

@@ -11,9 +11,7 @@ namespace Tests.BasicTests
         [Test]
         public void CodexLogExample()
         {
-            var primary = SetupCodexNodes(1)
+            var primary = SetupCodexNode(s => s.WithLogLevel(CodexLogLevel.Trace));
-                            .WithLogLevel(CodexLogLevel.Trace)
-                            .BringOnline()[0];
 
             primary.UploadFile(GenerateTestFile(5.MB()));
 
@@ -25,13 +23,8 @@ namespace Tests.BasicTests
         [Test]
         public void TwoMetricsExample()
         {
-            var group = SetupCodexNodes(2)
+            var group = SetupCodexNodes(2, s => s.EnableMetrics());
-                        .EnableMetrics()
+            var group2 = SetupCodexNodes(2, s => s.EnableMetrics());
-                        .BringOnline();
-
-            var group2 = SetupCodexNodes(2)
-                        .EnableMetrics()
-                        .BringOnline();
 
             var primary = group[0];
             var secondary = group[1];
@@ -50,29 +43,30 @@ namespace Tests.BasicTests
         [Test]
         public void MarketplaceExample()
         {
-            var primary = SetupCodexNodes(1)
+            var sellerInitialBalance = 234.TestTokens();
+            var buyerInitialBalance = 1000.TestTokens();
+
+            var seller = SetupCodexNode(s => s
+                            .WithLogLevel(CodexLogLevel.Trace)
                             .WithStorageQuota(11.GB())
-                            .EnableMarketplace(initialBalance: 234.TestTokens())
+                            .EnableMarketplace(sellerInitialBalance));
-                            .BringOnline()[0];
 
-            primary.Marketplace.AssertThatBalance(Is.EqualTo(234.TestTokens()));
+            seller.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance));
-
+            seller.Marketplace.MakeStorageAvailable(
-            var secondary = SetupCodexNodes(1)
-                            .EnableMarketplace(initialBalance: 1000.TestTokens())
-                            .BringOnline()[0];
-
-            primary.ConnectToPeer(secondary);
-
-            primary.Marketplace.MakeStorageAvailable(
                 size: 10.GB(),
                 minPricePerBytePerSecond: 1.TestTokens(),
                 maxCollateral: 20.TestTokens(),
                 maxDuration: TimeSpan.FromMinutes(3));
 
             var testFile = GenerateTestFile(10.MB());
-            var contentId = secondary.UploadFile(testFile);
 
-            secondary.Marketplace.RequestStorage(contentId,
+            var buyer = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace)
+                .WithBootstrapNode(seller)
+                .EnableMarketplace(buyerInitialBalance));
+            
+            var contentId = buyer.UploadFile(testFile);
+            buyer.Marketplace.RequestStorage(contentId,
                 pricePerBytePerSecond: 2.TestTokens(),
                 requiredCollateral: 10.TestTokens(),
                 minRequiredNumberOfNodes: 1,
@@ -81,12 +75,12 @@ namespace Tests.BasicTests
 
             Time.Sleep(TimeSpan.FromMinutes(1));
 
-            primary.Marketplace.AssertThatBalance(Is.LessThan(234.TestTokens()), "Collateral was not placed.");
+            seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
 
             Time.Sleep(TimeSpan.FromMinutes(2));
 
-            primary.Marketplace.AssertThatBalance(Is.GreaterThan(234.TestTokens()), "Storer was not paid for storage.");
+            seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
-            secondary.Marketplace.AssertThatBalance(Is.LessThan(1000.TestTokens()), "Contractor was not charged for storage.");
+            buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");
         }
     }
 }

Tests/BasicTests/OneClientTests.cs

@@ -9,7 +9,7 @@ namespace Tests.BasicTests
         [Test]
         public void OneClientTest()
         {
-            var primary = SetupCodexNodes(1).BringOnline()[0];
+            var primary = SetupCodexNode();
 
             PerformOneClientTest(primary);
         }
@@ -17,11 +17,11 @@ namespace Tests.BasicTests
         [Test]
         public void RestartTest()
         {
-            var group = SetupCodexNodes(1).BringOnline();
+            var primary = SetupCodexNode();
 
-            var setup = group.BringOffline();
+            var setup = primary.BringOffline();
 
-            var primary = setup.BringOnline()[0];
+            primary = BringOnline(setup)[0];
 
             PerformOneClientTest(primary);
         }

Tests/BasicTests/PeerTests.cs

@@ -0,0 +1,85 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using NUnit.Framework;
+
+namespace Tests.BasicTests
+{
+    [TestFixture]
+    public class PeerTests : DistTest
+    {
+        [Test]
+        public void TwoNodes()
+        {
+            var primary = SetupCodexBootstrapNode();
+            var secondary = SetupCodexNode(s => s.WithBootstrapNode(primary));
+
+            primary.ConnectToPeer(secondary); // TODO REMOVE THIS: This is required for the switchPeers to show up.
+
+            // This is required for the enginePeers to show up.
+            //var file = GenerateTestFile(10.MB());
+            //var contentId = primary.UploadFile(file);
+            //var file2 = secondary.DownloadContent(contentId);
+            //file.AssertIsEqual(file2);
+
+            AssertKnowEachother(primary, secondary);
+        }
+
+        [TestCase(2)]
+        [TestCase(3)]
+        [TestCase(10)]
+        public void VariableNodes(int number)
+        {
+            var bootstrap = SetupCodexBootstrapNode();
+            var nodes = SetupCodexNodes(number, s => s.WithBootstrapNode(bootstrap));
+
+            var file = GenerateTestFile(10.MB());
+            var contentId = nodes.First().UploadFile(file);
+            var file2 = nodes.Last().DownloadContent(contentId);
+            file.AssertIsEqual(file2);
+
+            // <TODO REMOVE THIS>
+            foreach (var node in nodes) bootstrap.ConnectToPeer(node);
+            for (var x = 0; x < number; x++)
+            {
+                for (var y = x + 1; y < number; y++)
+                {
+                    nodes[x].ConnectToPeer(nodes[y]);
+                }
+            }
+            // </TODO REMOVE THIS>
+
+            foreach (var node in nodes) AssertKnowEachother(node, bootstrap);
+
+            for (var x = 0; x < number; x++)
+            {
+                for (var y = x + 1; y < number; y++)
+                {
+                    AssertKnowEachother(nodes[x], nodes[y]);
+                }
+            }
+        }
+
+        private void AssertKnowEachother(IOnlineCodexNode a, IOnlineCodexNode b)
+        {
+            AssertKnowEachother(a.GetDebugInfo(), b.GetDebugInfo());
+        }
+
+        private void AssertKnowEachother(CodexDebugResponse a, CodexDebugResponse b)
+        {
+            AssertKnows(a, b);
+            AssertKnows(b, a);
+        }
+
+        private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b)
+        {
+            //var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
+            var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId));
+
+            //Log.Debug($"Looking for {b.id} in engine-peers [{enginePeers}]");
+            Log.Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
+
+            //Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
+            Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found.");
+        }
+    }
+}

Tests/BasicTests/TwoClientTests.cs

@@ -10,7 +10,7 @@ namespace Tests.BasicTests
         [Test]
         public void TwoClientsOnePodTest()
         {
-            var group = SetupCodexNodes(2).BringOnline();
+            var group = SetupCodexNodes(2);
 
             var primary = group[0];
             var secondary = group[1];
@@ -21,9 +21,8 @@ namespace Tests.BasicTests
         [Test]
         public void TwoClientsTwoPodsTest()
         {
-            var primary = SetupCodexNodes(1).BringOnline()[0];
+            var primary = SetupCodexNode();
-
+            var secondary = SetupCodexNode();
-            var secondary = SetupCodexNodes(1).BringOnline()[0];
 
             PerformTwoClientTest(primary, secondary);
         }
@@ -32,13 +31,8 @@ namespace Tests.BasicTests
         [Ignore("Requires Location map to be configured for k8s cluster.")]
         public void TwoClientsTwoLocationsTest()
         {
-            var primary = SetupCodexNodes(1)
+            var primary = SetupCodexNode(s => s.At(Location.BensLaptop));
-                            .At(Location.BensLaptop)
+            var secondary = SetupCodexNode(s => s.At(Location.BensOldGamingMachine));
-                            .BringOnline()[0];
-
-            var secondary = SetupCodexNodes(1)
-                            .At(Location.BensOldGamingMachine)
-                            .BringOnline()[0];
 
             PerformTwoClientTest(primary, secondary);
         }

Tests/BasicTests/UploadTests.cs

@@ -1,30 +1,19 @@
 using DistTestCore;
-using KubernetesWorkflow;
 using NUnit.Framework;
+
 namespace Tests.ParallelTests
 {
     [TestFixture]
     public class UploadTests : DistTest
     {
-        [Test]
+        [TestCase(3, 50)]
-        public void ThreeNodeUploads()
+        [TestCase(5, 75)]
+        [TestCase(10, 25)]
+        [UseLongTimeouts]
+        public void ParallelUpload(int numberOfNodes, int filesizeMb)
         {
-            ParallelUpload(3, 50.MB());
+            var group = SetupCodexNodes(numberOfNodes);
-        }
+            var host = SetupCodexNode();
-        [Test]
-        public void FiveNodeUploads()
-        {
-            ParallelUpload(5, 750.MB());
-        }
-        [Test]
-        public void TenNodeUploads()
-        {
-            ParallelUpload(10, 25.MB());
-        }
-        void ParallelUpload(int numberOfNodes, ByteSize filesize)
-        {
-            var group = SetupCodexNodes(numberOfNodes).BringOnline();
-            var host = SetupCodexNodes(1).BringOnline()[0];
 
             foreach (var node in group)
             {
@@ -36,7 +25,7 @@ namespace Tests.ParallelTests
 
             for (int i = 0; i < group.Count(); i++)
             {
-                testfiles.Add(GenerateTestFile(filesize));
+                testfiles.Add(GenerateTestFile(filesizeMb.MB()));
                 var n = i;
                 contentIds.Add(Task.Run(() => { return host.UploadFile(testfiles[n]); }));
             }

Tests/DurabilityTests/DurabilityTests.cs

@@ -1,4 +1,5 @@
 using DistTestCore;
+using DistTestCore.Codex;
 using NUnit.Framework;
 using Utils;
 
@@ -10,14 +11,14 @@ namespace Tests.DurabilityTests
         [Test]
         public void BootstrapNodeDisappearsTest()
         {
-            var bootstrapNode = SetupCodexNodes(1).BringOnline();
+            var bootstrapNode = SetupCodexNode();
-            var group = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode[0]).BringOnline();
+            var group = SetupCodexNodes(2, s => s.WithBootstrapNode(bootstrapNode));
             var primary = group[0];
             var secondary = group[1];
 
             // There is 1 minute of time for the nodes to connect to each other.
             // (Should be easy, they're in the same pod.)
-            Time.Sleep(TimeSpan.FromMinutes(1));
+            Time.Sleep(TimeSpan.FromMinutes(6));
             bootstrapNode.BringOffline();
 
             var file = GenerateTestFile(10.MB());
@@ -30,10 +31,10 @@ namespace Tests.DurabilityTests
         [Test]
         public void DataRetentionTest()
         {
-            var bootstrapNode = SetupCodexNodes(1).BringOnline()[0];
+            var bootstrapNode = SetupCodexNode(s => s.WithLogLevel(CodexLogLevel.Trace));
 
-            var startGroup = SetupCodexNodes(2).WithBootstrapNode(bootstrapNode).BringOnline();
+            var startGroup = SetupCodexNodes(2, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode));
-            var finishGroup = SetupCodexNodes(10).WithBootstrapNode(bootstrapNode).BringOnline();
+            var finishGroup = SetupCodexNodes(10, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode));
 
             var file = GenerateTestFile(10.MB());
 

Utils/DebugStack.cs

@@ -0,0 +1,12 @@
+using System.Diagnostics;
+
+namespace Utils
+{
+    public class DebugStack
+    {
+        public static string GetCallerName(int skipFrames = 0)
+        {
+            return new StackFrame(2 + skipFrames, true).GetMethod()!.Name;
+        }
+    }
+}