test: testing slot expiry

The test is mostly working, except that the node needs to be updated to call the onReservationAdded callback once availability marked unused

.github/workflows/dist-tests.yaml

@@ -15,19 +15,19 @@ on:
   workflow_dispatch:
     inputs:
       branch:
-        description: Branch
+        description: Branch (master)
         required: false
         type: string
       source:
-        description: Repository with tests
+        description: Repository with tests (current)
         required: false
         type: string
       nameprefix:
-        description: Runner job/pod name prefix
+        description: Runner prefix (cs-codex-dist-tests)
         required: false
         type: string
       namespace:
-        description: Kubernetes namespace for runner
+        description: Runner namespace (cs-codex-dist-tests)
         required: false
         type: string
 
@@ -56,6 +56,8 @@ jobs:
           [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
           [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
           [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
 
       - name: Kubectl - Install ${{ env.KUBE_VERSION }}
         uses: azure/setup-kubectl@v3
@@ -69,5 +71,4 @@ jobs:
 
       - name: Kubectl - Create Job
         run: |
-          export RUNID=$(date +%Y%m%d-%H%M%S)
           envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -

CodexNetDeployer/CodexNodeStarter.cs

@@ -8,16 +8,14 @@ namespace CodexNetDeployer
     public class CodexNodeStarter
     {
         private readonly Configuration config;
-        private readonly WorkflowCreator workflowCreator;
         private readonly TestLifecycle lifecycle;
         private readonly GethStartResult gethResult;
         private string bootstrapSpr = "";
         private int validatorsLeft;
 
-        public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
+        public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
         {
             this.config = config;
-            this.workflowCreator = workflowCreator;
             this.lifecycle = lifecycle;
             this.gethResult = gethResult;
             validatorsLeft = numberOfValidators;
@@ -26,7 +24,7 @@ namespace CodexNetDeployer
         public RunningContainer? Start(int i)
         {
             Console.Write($" - {i} = ");
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var workflowStartup = new StartupConfig();
             workflowStartup.Add(gethResult);
             workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));

CodexNetDeployer/Configuration.cs

@@ -1,5 +1,4 @@
 using ArgsUniform;
-using DistTestCore;
 using DistTestCore.Codex;
 
 namespace CodexNetDeployer
@@ -46,9 +45,11 @@ namespace CodexNetDeployer
 
         [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
         public bool RecordMetrics { get; set; } = false;
-       
-        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
 
+        [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
+            "set this option to override the label value.")]
+        public string TestsTypePodLabel { get; set; } = "continuous-tests";
+       
         public List<string> Validate()
         {
             var errors = new List<string>();

CodexNetDeployer/Deployer.cs

@@ -21,7 +21,7 @@ namespace CodexNetDeployer
         public CodexDeployment Deploy()
         {
             Log("Initializing...");
-            var (workflowCreator, lifecycle) = CreateFacilities();
+            var lifecycle = CreateTestLifecycle();
 
             Log("Preparing configuration...");
             // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
@@ -30,7 +30,7 @@ namespace CodexNetDeployer
             setup.MetricsEnabled = config.RecordMetrics;
 
             Log("Creating Geth instance and deploying contracts...");
-            var gethStarter = new GethStarter(lifecycle, workflowCreator);
+            var gethStarter = new GethStarter(lifecycle);
             var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
 
             Log("Geth started. Codex contracts deployed.");
@@ -44,7 +44,7 @@ namespace CodexNetDeployer
             Log("Starting Codex nodes...");
 
             // Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
-            var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, gethResults, config.NumberOfValidators!.Value);
+            var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
             var codexContainers = new List<RunningContainer>();
             for (var i = 0; i < config.NumberOfCodexNodes; i++)
             {
@@ -57,7 +57,7 @@ namespace CodexNetDeployer
             return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
         }
 
-        private (WorkflowCreator, TestLifecycle) CreateFacilities()
+        private TestLifecycle CreateTestLifecycle()
         {
             var kubeConfig = GetKubeConfig(config.KubeConfigFile);
 
@@ -68,19 +68,10 @@ namespace CodexNetDeployer
                 logDebug: false,
                 dataFilesPath: "notUsed",
                 codexLogLevel: config.CodexLogLevel,
-                runnerLocation: config.RunnerLocation
+                k8sNamespacePrefix: config.KubeNamespace
             );
 
-            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: config.KubeNamespace,
-                kubeConfigFile: kubeConfig,
-                operationTimeout: timeset.K8sOperationTimeout(),
-                retryDelay: timeset.WaitForK8sServiceDelay());
-
-            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
-            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
-
-            return (workflowCreator, lifecycle);
+            return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
         }
 
         private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
@@ -88,7 +79,7 @@ namespace CodexNetDeployer
             if (!setup.MetricsEnabled) return null;
 
             Log("Starting metrics service...");
-            var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray());
+            var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
             return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
         }
 

CodexNetDeployer/Program.cs

@@ -1,6 +1,5 @@
 using ArgsUniform;
 using CodexNetDeployer;
-using DistTestCore;
 using DistTestCore.Codex;
 using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
@@ -17,11 +16,6 @@ public class Program
         var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
         var config = uniformArgs.Parse(true);
         
-        if (args.Any(a => a == "--external"))
-        {
-            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
-        }
-
         var errors = config.Validate();
         if (errors.Any())
         {
@@ -33,10 +27,10 @@ public class Program
         }
 
         Console.WriteLine("Using images:" + nl +
-            $"\tCodex image: '{CodexContainerRecipe.DockerImage}'" + nl +
-            $"\tCodex Contracts image: '{CodexContractsContainerRecipe.DockerImage}'" + nl +
-            $"\tPrometheus image: '{PrometheusContainerRecipe.DockerImage}'" + nl +
-            $"\tGeth image: '{GethContainerRecipe.DockerImage}'" + nl);
+            $"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl +
+            $"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl +
+            $"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl +
+            $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl);
 
         if (!args.Any(a => a == "-y"))
         {

CodexNetDeployer/deploy-continuous-testnet.sh

@@ -3,10 +3,11 @@ dotnet run \
     --kube-namespace=codex-continuous-tests \
     --nodes=5 \
     --validators=3 \
+    --log-level=Trace \
     --storage-quota=2048 \
     --storage-sell=1024 \
     --min-price=1024 \
     --max-collateral=1024 \
     --max-duration=3600000 \
-    --block-ttl=120
-
+    --block-ttl=300 \
+    --record-metrics=true

CodexNetDownloader/Configuration.cs

@@ -1,5 +1,4 @@
 using ArgsUniform;
-using DistTestCore;
 using DistTestCore.Codex;
 
 namespace CodexNetDownloader
@@ -16,7 +15,5 @@ namespace CodexNetDownloader
         public string KubeConfigFile { get; set; } = "null";
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
-
-        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
     }
 }

CodexNetDownloader/Program.cs

@@ -15,17 +15,12 @@ public class Program
         var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(PrintHelp, args);
         var config = uniformArgs.Parse(true);
 
-        if (args.Any(a => a == "--external"))
-        {
-            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
-        }
-
         config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
 
         if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
 
         var k8sFactory = new K8sFactory();
-        var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+        var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
 
         foreach (var container in config.CodexDeployment.CodexContainers)
         {

ContinuousTests/CodexAccessFactory.cs

@@ -12,7 +12,7 @@ namespace ContinuousTests
             return containers.Select(container =>
             {
                 var address = container.ClusterExternalAddress;
-                if (config.RunnerLocation == TestRunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
+                if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
                 return new CodexAccess(log, container, timeSet, address);
             }).ToArray();
         }

ContinuousTests/Configuration.cs

@@ -25,9 +25,12 @@ namespace ContinuousTests
         [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
         public bool StopOnFailure { get; set; } = false;
 
+        [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
+        public bool DownloadContainerLogs { get; set; } = false;
+
         public CodexDeployment CodexDeployment { get; set; } = null!;
 
-        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
+        public RunnerLocation RunnerLocation { get; set; }
     }
 
     public class ConfigLoader
@@ -39,10 +42,7 @@ namespace ContinuousTests
             var result = uniformArgs.Parse(true);
             
             result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
-            if (args.Any(a => a == "--external"))
-            {
-                result.RunnerLocation = TestRunnerLocation.ExternalToCluster;
-            }
+            result.RunnerLocation = RunnerLocationUtils.DetermineRunnerLocation(result.CodexDeployment.CodexContainers.First());
 
             return result;
         }
@@ -57,9 +57,10 @@ namespace ContinuousTests
         private static void PrintHelp()
         {
             var nl = Environment.NewLine;
-            Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
+            Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
+                "The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
 
-            Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
+            Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
                 "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
         }
     }

ContinuousTests/ContinuousLogDownloader.cs

@@ -0,0 +1,99 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+
+namespace ContinuousTests
+{
+    public class ContinuousLogDownloader
+    {
+        private readonly TestLifecycle lifecycle;
+        private readonly CodexDeployment deployment;
+        private readonly string outputPath;
+        private readonly CancellationToken cancelToken;
+
+        public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken)
+        {
+            this.lifecycle = lifecycle;
+            this.deployment = deployment;
+            this.outputPath = outputPath;
+            this.cancelToken = cancelToken;
+        }
+
+        public void Run()
+        {
+            while (!cancelToken.IsCancellationRequested)
+            {
+                UpdateLogs();
+
+                cancelToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(15));
+            }
+
+            // After testing has stopped, we wait a little bit and fetch the logs one more time.
+            // If our latest fetch was not recent, interesting test-related log activity might
+            // not have been captured yet.
+            Thread.Sleep(TimeSpan.FromSeconds(10));
+            UpdateLogs();
+        }
+
+        private void UpdateLogs()
+        {
+            foreach (var container in deployment.CodexContainers)
+            {
+                UpdateLog(container);
+            }
+        }
+
+        private void UpdateLog(RunningContainer container)
+        {
+            var filepath = Path.Combine(outputPath, GetLogName(container));
+            if (!File.Exists(filepath))
+            {
+                File.WriteAllLines(filepath, new[] { container.Name });
+            }
+
+            var appender = new LogAppender(filepath);
+
+            lifecycle.CodexStarter.DownloadLog(container, appender);
+        }
+
+        private static string GetLogName(RunningContainer container)
+        {
+            return container.Name
+                .Replace("<", "")
+                .Replace(">", "")
+                + ".log";
+        }
+    }
+
+    public class LogAppender : ILogHandler
+    {
+        private readonly string filename;
+
+        public LogAppender(string filename)
+        {
+            this.filename = filename;
+        }
+
+        public void Log(Stream log)
+        {
+            using var reader = new StreamReader(log);
+            var lines = File.ReadAllLines(filename);
+            var lastLine = lines.Last();
+            var recording = lines.Length < 3;
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                if (recording)
+                {
+                    File.AppendAllLines(filename, new[] { line });
+                }
+                else
+                {
+                    recording = line == lastLine;
+                }
+
+                line = reader.ReadLine();
+            }
+        }
+    }
+}

ContinuousTests/ContinuousTestRunner.cs

@@ -24,12 +24,14 @@ namespace ContinuousTests
             startupChecker.Check();
 
             var taskFactory = new TaskFactory();
-            var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
+            var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "Overview");
             overviewLog.Log("Continuous tests starting...");
             var allTests = testFactory.CreateTests();
 
             ClearAllCustomNamespaces(allTests, overviewLog);
 
+            StartLogDownloader(taskFactory);
+
             var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
 
             foreach (var testLoop in testLoops)
@@ -58,8 +60,21 @@ namespace ContinuousTests
             if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
 
             log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
-            var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
-            workflowCreator.CreateWorkflow().DeleteTestResources();
+            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log);
+            lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources();
+        }
+
+        private void StartLogDownloader(TaskFactory taskFactory)
+        {
+            if (!config.DownloadContainerLogs) return;
+
+            var path = Path.Combine(config.LogPath, "containers");
+            if (!Directory.Exists(path)) Directory.CreateDirectory(path);
+
+            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
+            var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
+
+            taskFactory.Run(downloader.Run);
         }
     }
 }

ContinuousTests/K8sFactory.cs

@@ -1,13 +1,12 @@
 using DistTestCore.Codex;
 using DistTestCore;
-using KubernetesWorkflow;
 using Logging;
 
 namespace ContinuousTests
 {
     public class K8sFactory
     {
-        public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation)
+        public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log)
         {
             var kubeConfig = GetKubeConfig(kubeConfigFile);
             var lifecycleConfig = new DistTestCore.Configuration
@@ -17,19 +16,10 @@ namespace ContinuousTests
                 logDebug: false,
                 dataFilesPath: dataFilePath,
                 codexLogLevel: CodexLogLevel.Debug,
-                runnerLocation: runnerLocation
+                k8sNamespacePrefix: customNamespace
             );
 
-            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: customNamespace,
-                kubeConfigFile: kubeConfig,
-                operationTimeout: timeSet.K8sOperationTimeout(),
-            retryDelay: timeSet.WaitForK8sServiceDelay());
-
-            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
-            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator);
-
-            return (workflowCreator, lifecycle);
+            return new TestLifecycle(log, lifecycleConfig, timeSet, "continuous-tests", string.Empty);
         }
 
         private static string? GetKubeConfig(string kubeConfigFile)

ContinuousTests/NodeRunner.cs

@@ -40,8 +40,8 @@ namespace ContinuousTests
 
         public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
         {
-            var (workflowCreator, lifecycle) = CreateFacilities();
-            var flow = workflowCreator.CreateWorkflow();
+            var lifecycle = CreateTestLifecycle();
+            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
 
             try
             {
@@ -89,9 +89,9 @@ namespace ContinuousTests
             }
         }
 
-        private (WorkflowCreator, TestLifecycle) CreateFacilities()
+        private TestLifecycle CreateTestLifecycle()
         {
-            return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation);
+            return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log);
         }
     }
 }

ContinuousTests/SingleTestRun.cs

@@ -32,7 +32,7 @@ namespace ContinuousTests
             this.handle = handle;
             this.cancelToken = cancelToken;
             testName = handle.Test.GetType().Name;
-            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), testName);
+            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
 
             nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
             dataFolder = config.DataPath + "-" + Guid.NewGuid();
@@ -138,7 +138,7 @@ namespace ContinuousTests
         private void DownloadClusterLogs()
         {
             var k8sFactory = new K8sFactory();
-            var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+            var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
 
             foreach (var container in config.CodexDeployment.CodexContainers)
             {
@@ -221,7 +221,7 @@ namespace ContinuousTests
         private DistTestCore.Configuration CreateFileManagerConfiguration()
         {
             return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
-                CodexLogLevel.Error, config.RunnerLocation);
+                CodexLogLevel.Error, string.Empty);
         }
     }
 }

ContinuousTests/StartupChecker.cs

@@ -19,7 +19,7 @@ namespace ContinuousTests
 
         public void Check()
         {
-            var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
+            var log = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "StartupChecks");
             log.Log("Starting continuous test run...");
             log.Log("Checking configuration...");
             PreflightCheck(config);

ContinuousTests/Tests/MarketplaceTest.cs

@@ -60,40 +60,5 @@
 //                file.AssertIsEqual(result);
 //            });
 //        }
-
-//        private void WaitForContractToStart(CodexAccess codexAccess, string purchaseId)
-//        {
-//            var lastState = "";
-//            var waitStart = DateTime.UtcNow;
-//            var filesizeInMb = fileSize.SizeInBytes / (1024 * 1024);
-//            var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0);
-
-//            Log.Log($"{nameof(WaitForContractToStart)} for {Time.FormatDuration(maxWaitTime)}");
-//            while (lastState != "started")
-//            {
-//                CancelToken.ThrowIfCancellationRequested();
-
-//                var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId);
-//                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
-//                if (purchaseStatus != null && purchaseStatus.state != lastState)
-//                {
-//                    lastState = purchaseStatus.state;
-//                    Log.Log("Purchase status: " + statusJson);
-//                }
-
-//                Thread.Sleep(2000);
-
-//                if (lastState == "errored")
-//                {
-//                    Assert.Fail("Contract start failed: " + statusJson);
-//                }
-
-//                if (DateTime.UtcNow - waitStart > maxWaitTime)
-//                {
-//                    Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: {statusJson}");
-//                }
-//            }
-//            Log.Log("Contract started.");
-//        }
 //    }
 //}

ContinuousTests/reports/CodexTestNetReport-July2023.md

@@ -0,0 +1,38 @@
+# Codex Continuous Test-net Report
+Date: 02-08-2023
+
+Report for: 07-2023
+
+
+## Test-net Status
+- Start of month: Offline - faulted
+- End of month: Offline - faulted
+
+(Faulted: Tests fail with such frequency that the information gathered does not justify the cost of leaving the test-net running.)
+
+## Deployment Configuration
+Continous Test-net is deployed to the kubernetes cluster with the following configuration:
+
+5x Codex Nodes:
+- Log-level: Trace
+- Storage quota: 2048 MB
+- Storage sell: 1024 MB
+- Min price: 1024
+- Max collateral: 1024
+- Max duration: 3600000 seconds
+- Block-TTL: 120 seconds
+
+3 of these 5 nodes have:
+- Validator: true
+
+Kubernetes namespace: 'codex-continuous-tests'
+
+## Test Overview
+| Changes        | Test                | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | Status      | Results                                                                              |
+|----------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|--------------------------------------------------------------------------------------|
+| New in 07-2023 | Two-client test     | Every 30 seconds, two nodes are chosen at random. A 10 MB file is generated and uploaded to one. 10 seconds later, it is downloaded from the other. File contents are asserted to be equal.                                                                                                                                                                                                                                                                                                                                                                                    | Faulted     | Test reliably fails after 30 to 45 minutes. Both upload and download failures occur. |
+| New in 07-2023 | Transient-node test | Every 1 minute, a new, transient Codex node is started and bootstrapped against a random node of the test net. A 10 MB file is generated and uploaded to the transient node. The file is then downloaded from a second random test net node and file equality is asserted. After that, the transient node is shut down. 30 seconds later, a new transient Codex node is started and bootstrapped against a third (guaranteed different from first and second) node from the test net. The same file is then downloaded from the new transient node. File equality is asserted. | Not running | Test was not run because the previous more rudamentory test has faulted.             |
+
+## Action Points
+- Codex logs (from these long-running containers) are often incomplete when downloaded after a test failure. A reliable way of maintaining these logs is needed. The logs can quickly explode in size, even for test-nets that run only for a few hours.
+- The distributed testing setup can and has been used to reproduce the failure of the Two-client test in a local environment. Investigation is on-going.

ContinuousTests/run.sh

@@ -1,4 +1,6 @@
 dotnet run \
     --kube-config=/opt/kubeconfig.yaml \
     --codex-deployment=codex-deployment.json \
-    --stop=1 
+    --keep=1 \
+    --stop=1 \
+    --dl-logs=1

DistTestCore/BaseStarter.cs

@@ -1,18 +1,15 @@
-using KubernetesWorkflow;
-using Logging;
+using Logging;
 
 namespace DistTestCore
 {
     public class BaseStarter
     {
         protected readonly TestLifecycle lifecycle;
-        protected readonly WorkflowCreator workflowCreator;
         private Stopwatch? stopwatch;
 
-        public BaseStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+        public BaseStarter(TestLifecycle lifecycle)
         {
             this.lifecycle = lifecycle;
-            this.workflowCreator = workflowCreator;
         }
 
         protected void LogStart(string msg)

DistTestCore/Codex/CodexApiTypes.cs

@@ -1,5 +1,6 @@
 using KubernetesWorkflow;
 using Logging;
+using Newtonsoft.Json;
 using Utils;
 
 namespace DistTestCore.Codex
@@ -60,6 +61,16 @@ namespace DistTestCore.Codex
     {
         public string version { get; set; } = string.Empty;
         public string revision { get; set; } = string.Empty;
+
+        public bool IsValid()
+        {
+            return !string.IsNullOrEmpty(version) && !string.IsNullOrEmpty(revision);
+        }
+
+        public override string ToString()
+        {
+            return JsonConvert.SerializeObject(this);
+        }
     }
 
     public class CodexDebugPeerResponse

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -5,12 +5,7 @@ namespace DistTestCore.Codex
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-#if Arm64
-        public const string DockerImage = "codexstorage/nim-codex:sha-7227a4a";
-#else
-        //public const string DockerImage = "thatbenbierens/nim-codex:loopingyeah";
-        public const string DockerImage = "codexstorage/nim-codex:sha-7227a4a";
-#endif
+        public const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
         public const string MetricsPortTag = "metrics_port";
         public const string DiscoveryPortTag = "discovery-port";
 
@@ -18,15 +13,12 @@ namespace DistTestCore.Codex
         public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
         public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
 
-        public static string DockerImageOverride = string.Empty;
+        public override string AppName => "codex";
+        public override string Image { get; }
 
-        protected override string Image
+        public CodexContainerRecipe()
         {
-            get
-            {
-                if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride;
-                return DockerImage;
-            }
+            Image = GetDockerImage();
         }
 
         protected override void Initialize(StartupConfig startupConfig)
@@ -38,7 +30,7 @@ namespace DistTestCore.Codex
 
             AddEnvVar("CODEX_DATA_DIR", $"datadir{ContainerNumber}");
             AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
-            AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
+            AddEnvVar("CODEX_LOG_LEVEL", config.LogLevelWithTopics());
 
             // This makes the node announce itself to its local (pod) IP address.
             AddEnvVar("NAT_IP_AUTO", "true");
@@ -65,6 +57,14 @@ namespace DistTestCore.Codex
                 AddInternalPortAndVar("CODEX_METRICS_PORT", tag: MetricsPortTag);
             }
 
+			if (config.SimulateProofFailures != null)
+            {
+                AddEnvVar("CODEX_SIMULATE_PROOF_FAILURES", config.SimulateProofFailures.ToString()!);
+            }
+			// if (config.EnableValidator == true)
+            // {
+            //     AddEnvVar("CODEX_VALIDATOR", "true");
+            // }
             if (config.MarketplaceConfig != null)
             {
                 var gethConfig = startupConfig.Get<GethStartResult>();
@@ -82,8 +82,15 @@ namespace DistTestCore.Codex
 
                 if (config.MarketplaceConfig.IsValidator)
                 {
-                    AddEnvVar("CODEX_VALIDATOR", "true");
+                   AddEnvVar("CODEX_VALIDATOR", "true");
                 }
+            }
+			// if (config.MarketplaceConfig != null) {
+            //     AddEnvVar("CODEX_PERSISTENCE", "true");
+            // }
+
+            if(!string.IsNullOrEmpty(config.NameOverride)) {
+                AddEnvVar("CODEX_NODENAME", config.NameOverride);
             }
         }
 
@@ -92,5 +99,12 @@ namespace DistTestCore.Codex
             if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value;
             return Index;
         }
+
+        private string GetDockerImage()
+        {
+            var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
+            if (!string.IsNullOrEmpty(image)) return image;
+            return DefaultDockerImage;
+        }
     }
 }

DistTestCore/Codex/CodexStartupConfig.cs

@@ -10,13 +10,26 @@ namespace DistTestCore.Codex
             LogLevel = logLevel;
         }
 
+        public string LogLevelWithTopics()
+        {
+            var level = LogLevel.ToString()!.ToUpperInvariant();
+            if (LogTopics != null && LogTopics.Count() > 0)
+            {
+                level = $"INFO;{level}: {string.Join(",", LogTopics.Where(s => !string.IsNullOrEmpty(s)))}";
+            }
+            return level;
+        }
+
         public string? NameOverride { get; set; }
         public Location Location { get; set; }
-        public CodexLogLevel LogLevel { get; }
+        public CodexLogLevel LogLevel { get; set; }
+        public string[]? LogTopics { get; set; }
         public ByteSize? StorageQuota { get; set; }
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
         public string? BootstrapSpr { get; set; }
         public int? BlockTTL { get; set; }
+        public uint? SimulateProofFailures { get; set; }
+        public bool? EnableValidator { get; set; }
     }
 }

DistTestCore/CodexNodeGroup.cs

@@ -14,12 +14,13 @@ namespace DistTestCore
     {
         private readonly TestLifecycle lifecycle;
 
-        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
         {
             this.lifecycle = lifecycle;
             Setup = setup;
             Containers = containers;
-            Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
+            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
+            Version = new CodexDebugVersionResponse();
         }
 
         public IOnlineCodexNode this[int index]
@@ -44,8 +45,9 @@ namespace DistTestCore
         }
 
         public CodexSetup Setup { get; private set; }
-        public RunningContainers Containers { get; private set; }
+        public RunningContainers[] Containers { get; private set; }
         public OnlineCodexNode[] Nodes { get; private set; }
+        public CodexDebugVersionResponse Version { get; private set; }
 
         public IEnumerator<IOnlineCodexNode> GetEnumerator()
         {
@@ -64,14 +66,17 @@ namespace DistTestCore
 
         public void EnsureOnline()
         {
-            foreach (var node in Nodes)
+            foreach (var node in Nodes) node.EnsureOnlineGetVersionResponse();
+            var versionResponses = Nodes.Select(n => n.Version);
+
+            var first = versionResponses.First();
+            if (!versionResponses.All(v => v.version == first.version && v.revision == first.revision))
             {
-                var debugInfo = node.CodexAccess.GetDebugInfo();
-                var nodePeerId = debugInfo.id;
-                var nodeName = node.CodexAccess.Container.Name;
-                lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
-                lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
+                throw new Exception("Inconsistent version information received from one or more Codex nodes: " +
+                    string.Join(",", versionResponses.Select(v => v.ToString())));
             }
+
+            Version = first;
         }
 
         private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)

DistTestCore/CodexSetup.cs

@@ -8,14 +8,29 @@ namespace DistTestCore
     {
         ICodexSetup WithName(string name);
         ICodexSetup At(Location location);
+        ICodexSetup WithLogLevel(CodexLogLevel level);
+        /// <summary>
+        /// Sets the log level for codex. The default level is INFO and the
+        /// log level is applied only to the supplied topics.
+        /// </summary>
+        ICodexSetup WithLogLevel(CodexLogLevel level, params string[] topics);
         ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
         ICodexSetup WithStorageQuota(ByteSize storageQuota);
+        ICodexSetup WithBlockTTL(TimeSpan duration);
         ICodexSetup EnableMetrics();
         ICodexSetup EnableMarketplace(TestToken initialBalance);
         ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
         ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator);
+		/// <summary>
+        /// Provides an invalid proof every N proofs
+        /// </summary>
+        ICodexSetup WithSimulateProofFailures(uint failEveryNProofs);
+        /// <summary>
+        /// Enables the validation module in the node
+        /// </summary>
+        // ICodexSetup WithValidator();
     }
-    
+
     public class CodexSetup : CodexStartupConfig, ICodexSetup
     {
         public int NumberOfNodes { get; }
@@ -44,12 +59,31 @@ namespace DistTestCore
             return this;
         }
 
+        public ICodexSetup WithLogLevel(CodexLogLevel level)
+        {
+            LogLevel = level;
+            return this;
+        }
+
+        public ICodexSetup WithLogLevel(CodexLogLevel level, params string[] topics)
+        {
+            LogLevel = level;
+            LogTopics = topics;
+            return this;
+        }
+
         public ICodexSetup WithStorageQuota(ByteSize storageQuota)
         {
             StorageQuota = storageQuota;
             return this;
         }
 
+        public ICodexSetup WithBlockTTL(TimeSpan duration)
+        {
+            BlockTTL = Convert.ToInt32(duration.TotalSeconds);
+            return this;
+        }
+
         public ICodexSetup EnableMetrics()
         {
             MetricsEnabled = true;
@@ -63,7 +97,7 @@ namespace DistTestCore
 
         public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther)
         {
-            return EnableMarketplace(initialBalance, initialEther, false);
+			return EnableMarketplace(initialBalance, initialEther, false);
         }
 
         public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator)
@@ -72,6 +106,18 @@ namespace DistTestCore
             return this;
         }
 
+        public ICodexSetup WithSimulateProofFailures(uint failEveryNProofs)
+        {
+            SimulateProofFailures = failEveryNProofs;
+            return this;
+        }
+
+        // public ICodexSetup WithValidator()
+        // {
+        //     EnableValidator = true;
+        //     return this;
+        // }
+
         public string Describe()
         {
             var args = string.Join(',', DescribeArgs());
@@ -80,9 +126,11 @@ namespace DistTestCore
 
         private IEnumerable<string> DescribeArgs()
         {
-            yield return $"LogLevel={LogLevel}";
+            yield return $"LogLevel={LogLevelWithTopics()}";
             if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
-            if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
+            if (StorageQuota != null) yield return $"StorageQuota={StorageQuota}";
+            if (SimulateProofFailures != null) yield return $"SimulateProofFailures={SimulateProofFailures}";
+            if (MarketplaceConfig != null) yield return $"IsValidator={MarketplaceConfig.IsValidator}";
         }
     }
 }

DistTestCore/CodexStarter.cs

@@ -8,8 +8,8 @@ namespace DistTestCore
 {
     public class CodexStarter : BaseStarter
     {
-        public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public CodexStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
@@ -20,8 +20,10 @@ namespace DistTestCore
             LogSeparator();
             LogStart($"Starting {codexSetup.Describe()}...");
             var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
+            gethStartResult = lifecycle.GethStarter.BringOnlineValidatorFor(codexSetup, gethStartResult);
 
             var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
+
             var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
 
             var metricAccessFactory = CollectMetrics(codexSetup, containers);
@@ -29,9 +31,16 @@ namespace DistTestCore
             var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
 
             var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
-            var podInfo = group.Containers.RunningPod.PodInfo;
-            LogEnd($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers.First().Recipe.Image}' at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}");
+            lifecycle.SetCodexVersion(group.Version);
+
+            var nl = Environment.NewLine;
+            var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
+            LogEnd($"Started {codexSetup.NumberOfNodes} nodes " +
+                $"of image '{containers.Containers().First().Recipe.Image}' " +
+                $"and version '{group.Version}'{nl}" +
+                podInfos);
             LogSeparator();
+
             return group;
         }
 
@@ -39,7 +48,7 @@ namespace DistTestCore
         {
             LogStart($"Stopping {group.Describe()}...");
             var workflow = CreateWorkflow();
-            workflow.Stop(group.Containers);
+            foreach (var c in group.Containers) workflow.Stop(c);
             RunningGroups.Remove(group);
             LogEnd("Stopped.");
         }
@@ -58,7 +67,7 @@ namespace DistTestCore
             workflow.DownloadContainerLog(container, logHandler);
         }
 
-        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers containers)
+        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
         {
             if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
 
@@ -75,23 +84,45 @@ namespace DistTestCore
             return startupConfig;
         }
 
-        private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
+        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
         {
-            var workflow = CreateWorkflow();
-            return workflow.Start(numberOfNodes, location, new CodexContainerRecipe(), startupConfig);
+            var result = new List<RunningContainers>();
+            var recipe = new CodexContainerRecipe();
+            for (var i = 0; i < numberOfNodes; i++)
+            {
+                var workflow = CreateWorkflow();
+                result.Add(workflow.Start(1, location, recipe, startupConfig));
+            }
+            return result.ToArray();
         }
 
-        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory)
+        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
             RunningGroups.Add(group);
-            Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
+
+            try
+            {
+                Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
+            }
+            catch
+            {
+                CodexNodesNotOnline(runningContainers);
+                throw;
+            }
+
             return group;
         }
 
+        private void CodexNodesNotOnline(RunningContainers[] runningContainers)
+        {
+            Log("Codex nodes failed to start");
+            foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
+        }
+
         private StartupWorkflow CreateWorkflow()
         {
-            return workflowCreator.CreateWorkflow();
+            return lifecycle.WorkflowCreator.CreateWorkflow();
         }
 
         private void LogSeparator()

DistTestCore/Configuration.cs

@@ -1,5 +1,6 @@
 using DistTestCore.Codex;
 using KubernetesWorkflow;
+using System.Net.NetworkInformation;
 using Utils;
 
 namespace DistTestCore
@@ -11,7 +12,8 @@ namespace DistTestCore
         private readonly bool logDebug;
         private readonly string dataFilesPath;
         private readonly CodexLogLevel codexLogLevel;
-        private readonly TestRunnerLocation runnerLocation;
+        private readonly string k8sNamespacePrefix;
+        private RunnerLocation? runnerLocation = null;
 
         public Configuration()
         {
@@ -20,23 +22,23 @@ namespace DistTestCore
             logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
             dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
             codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
-            runnerLocation = ParseEnum.Parse<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
+            k8sNamespacePrefix = "ct-";
         }
 
-        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation)
+        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, string k8sNamespacePrefix)
         {
             this.kubeConfigFile = kubeConfigFile;
             this.logPath = logPath;
             this.logDebug = logDebug;
             this.dataFilesPath = dataFilesPath;
             this.codexLogLevel = codexLogLevel;
-            this.runnerLocation = runnerLocation;
+            this.k8sNamespacePrefix = k8sNamespacePrefix;
         }
 
         public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
         {
             return new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: "ct-",
+                k8sNamespacePrefix: k8sNamespacePrefix,
                 kubeConfigFile: kubeConfigFile,
                 operationTimeout: timeSet.K8sOperationTimeout(),
                 retryDelay: timeSet.WaitForK8sServiceDelay()
@@ -58,14 +60,14 @@ namespace DistTestCore
             return codexLogLevel;
         }
 
-        public TestRunnerLocation GetTestRunnerLocation()
-        {
-            return runnerLocation;
-        }
-
         public Address GetAddress(RunningContainer container)
         {
-            if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster)
+            if (runnerLocation == null)
+            {
+                runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container);
+            }
+            
+            if (runnerLocation == RunnerLocation.InternalToCluster)
             {
                 return container.ClusterInternalAddress;
             }
@@ -87,9 +89,56 @@ namespace DistTestCore
         }
     }
 
-    public enum TestRunnerLocation
+    public enum RunnerLocation
     {
         ExternalToCluster,
         InternalToCluster,
     }
+
+    public static class RunnerLocationUtils
+    {
+        private static bool alreadyDidThat = false;
+
+        public static RunnerLocation DetermineRunnerLocation(RunningContainer container)
+        {
+            // We want to be sure we don't ping more often than strictly necessary.
+            // If we have already determined the location during this application
+            // lifetime, don't do it again.
+            if (alreadyDidThat) throw new Exception("We already did that.");
+            alreadyDidThat = true;
+
+            if (PingHost(container.Pod.PodInfo.Ip))
+            {
+                return RunnerLocation.InternalToCluster;
+            }
+            if (PingHost(Format(container.ClusterExternalAddress)))
+            {
+                return RunnerLocation.ExternalToCluster;
+            }
+
+            throw new Exception("Unable to determine runner location.");
+        }
+
+        private static string Format(Address host)
+        {
+            return host.Host
+                .Replace("http://", "")
+                .Replace("https://", "");
+        }
+
+        private static bool PingHost(string host)
+        {
+            try
+            {
+                using var pinger = new Ping();
+                PingReply reply = pinger.Send(host);
+                return reply.Status == IPStatus.Success;
+            }
+            catch (PingException)
+            {
+            }
+
+            return false;
+        }
+    }
 }

DistTestCore/DistTest.cs

@@ -13,9 +13,11 @@ namespace DistTestCore
     [Parallelizable(ParallelScope.All)]
     public abstract class DistTest
     {
+        private const string TestsType = "dist-tests";
         private readonly Configuration configuration = new Configuration();
         private readonly Assembly[] testAssemblies;
         private readonly FixtureLog fixtureLog;
+        private readonly StatusLog statusLog;
         private readonly object lifecycleLock = new object();
         private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
 
@@ -24,7 +26,10 @@ namespace DistTestCore
             var assemblies = AppDomain.CurrentDomain.GetAssemblies();
             testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
 
-            fixtureLog = new FixtureLog(configuration.GetLogConfig());
+            var logConfig = configuration.GetLogConfig();
+            var startTime = DateTime.UtcNow;
+            fixtureLog = new FixtureLog(logConfig, startTime);
+            statusLog = new StatusLog(logConfig, startTime);
 
             PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
             PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
@@ -36,13 +41,19 @@ namespace DistTestCore
         [OneTimeSetUp]
         public void GlobalSetup()
         {
+            fixtureLog.Log($"Codex Distributed Tests are starting...");
+            fixtureLog.Log($"Codex image: '{new CodexContainerRecipe().Image}'");
+            fixtureLog.Log($"CodexContracts image: '{new CodexContractsContainerRecipe().Image}'");
+            fixtureLog.Log($"Prometheus image: '{new PrometheusContainerRecipe().Image}'");
+            fixtureLog.Log($"Geth image: '{new GethContainerRecipe().Image}'");
+
             // Previous test run may have been interrupted.
             // Begin by cleaning everything up.
             try
             {
                 Stopwatch.Measure(fixtureLog, "Global setup", () =>
                 {
-                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
+                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), new PodLabels(TestsType, null!), string.Empty);
                     wc.CreateWorkflow().DeleteAllResources();
                 });                
             }
@@ -54,9 +65,6 @@ namespace DistTestCore
             }
 
             fixtureLog.Log("Global setup cleanup successful");
-            fixtureLog.Log($"Codex image: '{CodexContainerRecipe.DockerImage}'");
-            fixtureLog.Log($"Prometheus image: '{PrometheusContainerRecipe.DockerImage}'");
-            fixtureLog.Log($"Geth image: '{GethContainerRecipe.DockerImage}'");
         }
 
         [SetUp]
@@ -183,11 +191,14 @@ namespace DistTestCore
         private void CreateNewTestLifecycle()
         {
             var testName = GetCurrentTestName();
+            fixtureLog.WriteLogTag();
             Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
             {
                 lock (lifecycleLock)
                 {
-                    lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
+                    var testNamespace = Guid.NewGuid().ToString();
+                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), TestsType, testNamespace);
+                    lifecycles.Add(testName, lifecycle);
                 }
             });
         }
@@ -195,7 +206,10 @@ namespace DistTestCore
         private void DisposeTestLifecycle()
         {
             var lifecycle = Get();
-            fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
+            var testResult = GetTestResult();
+            var testDuration = lifecycle.GetTestDuration();
+            fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
+            statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetApplicationIds());
             Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
             {
                 lifecycle.Log.EndTest();

DistTestCore/DistTestCore.csproj

@@ -5,10 +5,6 @@
 	  <RootNamespace>DistTestCore</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>
-    <IsArm64 Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)' == 'Arm64'">true</IsArm64>
-  </PropertyGroup>
-  <PropertyGroup Condition="'$(IsArm64)'=='true'">
-    <DefineConstants>Arm64</DefineConstants>
   </PropertyGroup>
 
   <ItemGroup>

DistTestCore/GethStarter.cs

@@ -1,5 +1,4 @@
 using DistTestCore.Marketplace;
-using KubernetesWorkflow;
 
 namespace DistTestCore
 {
@@ -8,13 +7,13 @@ namespace DistTestCore
         private readonly MarketplaceNetworkCache marketplaceNetworkCache;
         private readonly GethCompanionNodeStarter companionNodeStarter;
 
-        public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public GethStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
             marketplaceNetworkCache = new MarketplaceNetworkCache(
-                new GethBootstrapNodeStarter(lifecycle, workflowCreator),
-                new CodexContractsStarter(lifecycle, workflowCreator));
-            companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator);
+                new GethBootstrapNodeStarter(lifecycle),
+                new CodexContractsStarter(lifecycle));
+            companionNodeStarter = new GethCompanionNodeStarter(lifecycle);
         }
 
         public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup)
@@ -31,6 +30,17 @@ namespace DistTestCore
             return CreateGethStartResult(marketplaceNetwork, companionNode);
         }
 
+        public GethStartResult BringOnlineValidatorFor(CodexSetup codexSetup, GethStartResult previousResult)
+        {
+            // allow marketplace and validator to be enabled on the same Codex node
+            if (previousResult.CompanionNode != null || (codexSetup.EnableValidator ?? false) == false) return previousResult;
+
+            var marketplaceNetwork = marketplaceNetworkCache.Get();
+            var companionNode = StartCompanionNode(codexSetup, marketplaceNetwork);
+
+            return CreateGethStartResult(marketplaceNetwork, companionNode);
+        }
+
         private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
         {
             if (marketplaceConfig.InitialTestTokens.Amount == 0) return;

DistTestCore/Helpers/AssertHelpers.cs

@@ -10,8 +10,7 @@ namespace DistTestCore.Helpers
         {
             try
             {
-                var c = constraint.Resolve();
-                Time.WaitUntil(() => c.ApplyTo(actual()).IsSuccess);
+                Time.WaitUntil(() => constraint.Resolve().ApplyTo(actual()).IsSuccess);
             }
             catch (TimeoutException)
             {

DistTestCore/Helpers/FullConnectivityHelper.cs

@@ -0,0 +1,211 @@
+using DistTestCore.Codex;
+using NUnit.Framework;
+using Utils;
+
+namespace DistTestCore.Helpers
+{
+    public interface IFullConnectivityImplementation
+    {
+        string Description();
+        string ValidateEntry(FullConnectivityHelper.Entry entry, FullConnectivityHelper.Entry[] allEntries);
+        FullConnectivityHelper.PeerConnectionState Check(FullConnectivityHelper.Entry from, FullConnectivityHelper.Entry to);
+    }
+
+    public class FullConnectivityHelper
+    {
+        private static string Nl = Environment.NewLine;
+        private readonly DistTest test;
+        private readonly IFullConnectivityImplementation implementation;
+
+        public FullConnectivityHelper(DistTest test, IFullConnectivityImplementation implementation)
+        {
+            this.test = test;
+            this.implementation = implementation;
+        }
+
+        public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullyConnected(nodes.ToArray());
+        }
+
+        private void AssertFullyConnected(IOnlineCodexNode[] nodes)
+        {
+            test.Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            var entries = CreateEntries(nodes);
+            var pairs = CreatePairs(entries);
+
+            RetryWhilePairs(pairs, () =>
+            {
+                CheckAndRemoveSuccessful(pairs);
+            });
+
+            if (pairs.Any())
+            {
+                var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
+
+                test.Log($"Connections failed:{Nl}{pairDetails}");
+
+                Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
+            }
+            else
+            {
+                test.Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
+            }
+        }
+
+        private static void RetryWhilePairs(List<Pair> pairs, Action action)
+        {
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
+            while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
+            {
+                action();
+
+                Time.Sleep(TimeSpan.FromSeconds(2));
+            }
+        }
+
+        private void CheckAndRemoveSuccessful(List<Pair> pairs)
+        {
+            // For large sets, don't try and do all of them at once.
+            var selectedPair = pairs.Take(20).ToArray();
+            var pairDetails = new List<string>();
+
+            foreach (var pair in selectedPair)
+            {
+                test.ScopedTestFiles(pair.Check);
+
+                if (pair.Success)
+                {
+                    pairDetails.AddRange(pair.GetResultMessages());
+                    pairs.Remove(pair);
+                }
+            }
+
+            test.Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
+        }
+
+        private Entry[] CreateEntries(IOnlineCodexNode[] nodes)
+        {
+            var entries = nodes.Select(n => new Entry(n)).ToArray();
+
+            var errors = entries
+                            .Select(e => implementation.ValidateEntry(e, entries))
+                            .Where(s => !string.IsNullOrEmpty(s))
+                            .ToArray();
+
+            if (errors.Any())
+            {
+                Assert.Fail("Some node entries failed to validate: " + string.Join(Nl, errors));
+            }
+
+            return entries;
+        }
+
+        private List<Pair> CreatePairs(Entry[] entries)
+        {
+            return CreatePairsIterator(entries).ToList();
+        }
+
+        private IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
+        {
+            for (var x = 0; x < entries.Length; x++)
+            {
+                for (var y = x + 1; y < entries.Length; y++)
+                {
+                    yield return new Pair(implementation, entries[x], entries[y]);
+                }
+            }
+        }
+
+        public class Entry
+        {
+            public Entry(IOnlineCodexNode node)
+            {
+                Node = node;
+                Response = node.GetDebugInfo();
+            }
+
+            public IOnlineCodexNode Node { get; }
+            public CodexDebugResponse Response { get; }
+
+            public override string ToString()
+            {
+                if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
+                return Response.id;
+            }
+        }
+
+        public enum PeerConnectionState
+        {
+            Unknown,
+            Connection,
+            NoConnection,
+        }
+
+        public class Pair
+        {
+            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
+            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
+            private readonly IFullConnectivityImplementation implementation;
+
+            public Pair(IFullConnectivityImplementation implementation, Entry a, Entry b)
+            {
+                this.implementation = implementation;
+                A = a;
+                B = b;
+            }
+
+            public Entry A { get; }
+            public Entry B { get; }
+            public PeerConnectionState AKnowsB { get; private set; }
+            public PeerConnectionState BKnowsA { get; private set; }
+            public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
+            public bool Inconclusive { get { return AKnowsB == PeerConnectionState.Unknown || BKnowsA == PeerConnectionState.Unknown; } }
+
+            public void Check()
+            {
+                aToBTime = Measure(() => AKnowsB = Check(A, B));
+                bToATime = Measure(() => BKnowsA = Check(B, A));
+            }
+
+            public override string ToString()
+            {
+                return $"[{string.Join(",", GetResultMessages())}]";
+            }
+
+            public string[] GetResultMessages()
+            {
+                var aName = A.ToString();
+                var bName = B.ToString();
+
+                return new[]
+                {
+                    $"[{aName} --> {bName}] = {AKnowsB} ({aToBTime.TotalSeconds} seconds)",
+                    $"[{aName} <-- {bName}] = {BKnowsA} ({bToATime.TotalSeconds} seconds)"
+                };
+            }
+
+            private static TimeSpan Measure(Action action)
+            {
+                var start = DateTime.UtcNow;
+                action();
+                return DateTime.UtcNow - start;
+            }
+
+            private PeerConnectionState Check(Entry from, Entry to)
+            {
+                Thread.Sleep(10);
+
+                try
+                {
+                    return implementation.Check(from, to);
+                }
+                catch
+                {
+                    // Didn't get a conclusive answer. Try again later.
+                    return PeerConnectionState.Unknown;
+                }
+            }
+        }
+    }
+}

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -1,257 +1,66 @@
 using DistTestCore.Codex;
-using NUnit.Framework;
-using Utils;
+using static DistTestCore.Helpers.FullConnectivityHelper;
 
 namespace DistTestCore.Helpers
 {
-    public class PeerConnectionTestHelpers
+    public class PeerConnectionTestHelpers : IFullConnectivityImplementation
     {
-        private readonly Random random = new Random();
-        private readonly DistTest test;
+        private readonly FullConnectivityHelper helper;
 
         public PeerConnectionTestHelpers(DistTest test)
         {
-            this.test = test;
+            helper = new FullConnectivityHelper(test, this);
         }
 
         public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
         {
-            var n = nodes.ToArray();
-
-            AssertFullyConnected(n);
-
-            for (int i = 0; i < 5; i++)
-            {
-                Time.Sleep(TimeSpan.FromSeconds(30));
-                AssertFullyConnected(n);
-            }
+            helper.AssertFullyConnected(nodes);
         }
 
-        private void AssertFullyConnected(IOnlineCodexNode[] nodes)
+        public string Description()
         {
-            test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
-            var entries = CreateEntries(nodes);
-            var pairs = CreatePairs(entries);
-
-            RetryWhilePairs(pairs, () =>
-            {
-                CheckAndRemoveSuccessful(pairs);
-            });
-
-            if (pairs.Any())
-            {
-                test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
-                Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
-                test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
-            }
-            else
-            {
-                test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
-            }
+            return "Peer Discovery";
         }
 
-        private static void RetryWhilePairs(List<Pair> pairs, Action action)
+        public string ValidateEntry(Entry entry, Entry[] allEntries)
         {
-            var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(30);
-            while (pairs.Any() && timeout > DateTime.UtcNow)
+            var result = string.Empty;
+            foreach (var peer in entry.Response.table.nodes)
             {
-                action();
-
-                if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(2));
-            }
-        }
-
-        private void CheckAndRemoveSuccessful(List<Pair> pairs)
-        {
-            var checkTasks = pairs.Select(p => Task.Run(() =>
-            {
-                ApplyRandomDelay();
-                p.Check();
-            })).ToArray();
-
-            Task.WaitAll(checkTasks);
-
-            foreach (var pair in pairs.ToArray())
-            {
-                if (pair.Success)
+                var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
+                if (expected != peer.address)
                 {
-                    test.Log(pair.GetMessage());
-                    pairs.Remove(pair);
+                    result += $"Node:{entry.Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'. ";
                 }
             }
+            return result;
         }
 
-        private static Entry[] CreateEntries(IOnlineCodexNode[] nodes)
+        public PeerConnectionState Check(Entry from, Entry to)
         {
-            var entries = nodes.Select(n => new Entry(n)).ToArray();
-            var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
+            var peerId = to.Response.id;
 
-            if (incorrectDiscoveryEndpoints.Any())
+            var response = from.Node.GetDebugPeer(peerId);
+            if (!response.IsPeerFound)
             {
-                Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
-                    string.Join(Environment.NewLine, incorrectDiscoveryEndpoints));
+                return PeerConnectionState.NoConnection;
             }
-
-            return entries;
+            if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
+            {
+                return PeerConnectionState.Connection;
+            }
+            return PeerConnectionState.Unknown;
         }
 
-        private static List<Pair> CreatePairs(Entry[] entries)
+        private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
         {
-            return CreatePairsIterator(entries).ToList();
-        }
+            var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
+            if (peer == null) return $"peerId: {node.peerId} is not known.";
 
-        private static IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
-        {
-            for (var x = 0; x < entries.Length; x++)
-            {
-                for (var y = x + 1; y < entries.Length; y++)
-                {
-                    yield return new Pair(entries[x], entries[y]);
-                }
-            }
-        }
-
-        private void ApplyRandomDelay()
-        {
-            // Calling all the nodes all at the same time is not exactly nice.
-            Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000)));
-        }
-
-        public class Entry
-        {
-            public Entry(IOnlineCodexNode node)
-            {
-                Node = node;
-                Response = node.GetDebugInfo();
-            }
-
-            public IOnlineCodexNode Node { get; }
-            public CodexDebugResponse Response { get; }
-
-            public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
-            {
-                foreach (var peer in Response.table.nodes)
-                {
-                    var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
-                    if (expected != peer.address)
-                    {
-                        yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'";
-                    }
-                }
-            }
-
-            public override string ToString()
-            {
-                if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
-                return Response.id;
-            }
-
-            private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
-            {
-                var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
-                if (peer == null) return $"peerId: {node.peerId} is not known.";
-
-                var n = (OnlineCodexNode)peer.Node;
-                var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
-                var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
-                return $"{ip}:{discPort.Number}";
-            }
-        }
-
-        public enum PeerConnectionState
-        {
-            Unknown,
-            Connection,
-            NoConnection,
-        }
-
-        public class Pair
-        {
-            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
-            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
-
-            public Pair(Entry a, Entry b)
-            {
-                A = a;
-                B = b;
-            }
-
-            public Entry A { get; }
-            public Entry B { get; }
-            public PeerConnectionState AKnowsB { get; private set; }
-            public PeerConnectionState BKnowsA { get; private set; }
-            public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
-
-            public void Check()
-            {
-                aToBTime = Measure(() => AKnowsB = Knows(A, B));
-                bToATime = Measure(() => BKnowsA = Knows(B, A));
-            }
-
-            public string GetMessage()
-            {
-                return GetResultMessage() + GetTimePostfix();
-            }
-
-            public override string ToString()
-            {
-                return $"[{GetMessage()}]";
-            }
-
-            private string GetResultMessage()
-            {
-                var aName = A.ToString();
-                var bName = B.ToString();
-
-                if (Success)
-                {
-                    return $"{aName} and {bName} know each other.";
-                }
-
-                return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}";
-            }
-
-            private string GetTimePostfix()
-            {
-                var aName = A.ToString();
-                var bName = B.ToString();
-
-                return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
-            }
-
-            private static TimeSpan Measure(Action action)
-            {
-                var start = DateTime.UtcNow;
-                action();
-                return DateTime.UtcNow - start;
-            }
-
-            private PeerConnectionState Knows(Entry a, Entry b)
-            {
-                lock (a)
-                {
-                    var peerId = b.Response.id;
-
-                    try
-                    {
-                        var response = a.Node.GetDebugPeer(peerId);
-                        if (!response.IsPeerFound)
-                        {
-                            return PeerConnectionState.NoConnection;
-                        }
-                        if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
-                        {
-                            return PeerConnectionState.Connection;
-                        }
-                    }
-                    catch
-                    {
-                    }
-
-                    // Didn't get a conclusive answer. Try again later.
-                    return PeerConnectionState.Unknown;
-                }
-            }
+            var n = (OnlineCodexNode)peer.Node;
+            var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
+            var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
+            return $"{ip}:{discPort.Number}";
         }
     }
 }

DistTestCore/Helpers/PeerDownloadTestHelpers.cs

@@ -1,74 +1,63 @@
-using DistTestCore.Codex;
-using NUnit.Framework;
+using static DistTestCore.Helpers.FullConnectivityHelper;
 
 namespace DistTestCore.Helpers
 {
-    public class PeerDownloadTestHelpers
+    public class PeerDownloadTestHelpers : IFullConnectivityImplementation
     {
+        private readonly FullConnectivityHelper helper;
         private readonly DistTest test;
+        private ByteSize testFileSize;
 
         public PeerDownloadTestHelpers(DistTest test)
         {
+            helper = new FullConnectivityHelper(test, this);
+            testFileSize = 1.MB();
             this.test = test;
         }
 
         public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
         {
-            test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
-            var start = DateTime.UtcNow;
-
-            foreach (var node in nodes)
-            {
-                var uploader = node;
-                var downloaders = nodes.Where(n => n != uploader).ToArray();
-
-                test.ScopedTestFiles(() =>
-                {
-                    PerformTest(uploader, downloaders, testFileSize);
-                });
-            }
-            
-            test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
-            var timeTaken = DateTime.UtcNow - start;
-
-            AssertTimePerMB(timeTaken, nodes.Count(), testFileSize);
+            this.testFileSize = testFileSize;
+            helper.AssertFullyConnected(nodes);
         }
 
-        private void AssertTimePerMB(TimeSpan timeTaken, int numberOfNodes, ByteSize size)
+        public string Description()
         {
-            var numberOfDownloads = numberOfNodes * (numberOfNodes - 1);
-            var timePerDownload = timeTaken / numberOfDownloads;
-            float sizeInMB = size.ToMB();
-            var timePerMB = timePerDownload / sizeInMB;
-
-            test.Log($"Performed {numberOfDownloads} downloads of {size} in {timeTaken.TotalSeconds} seconds, for an average of {timePerMB.TotalSeconds} seconds per MB.");
-
-            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
+            return "Download Connectivity";
         }
 
-        private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
+        public string ValidateEntry(Entry entry, Entry[] allEntries)
         {
-            // Generate 1 test file per downloader.
-            var files = downloaders.Select(d => GenerateTestFile(uploader, d, testFileSize)).ToArray();
+            return string.Empty;
+        }
 
-            // Upload all the test files to the uploader.
-            var contentIds = files.Select(uploader.UploadFile).ToArray();
+        public PeerConnectionState Check(Entry from, Entry to)
+        {
+            var expectedFile = GenerateTestFile(from.Node, to.Node);
 
-            // Each downloader should retrieve its own test file.
-            for (var i = 0; i < downloaders.Length; i++)
+            var contentId = from.Node.UploadFile(expectedFile);
+
+            try
             {
-                var expectedFile = files[i];
-                var downloadedFile = downloaders[i].DownloadContent(contentIds[i], $"{expectedFile.Label}DOWNLOADED");
-
+                var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded");
                 expectedFile.AssertIsEqual(downloadedFile);
+                return PeerConnectionState.Connection;
             }
+            catch
+            {
+                // Should an exception occur during the download or file-content assertion,
+                // We consider that as no-connection for the purpose of this test.
+                return PeerConnectionState.NoConnection;
+            }
+
+            // Should an exception occur during upload, then this try is inconclusive and we try again next loop.
         }
 
-        private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader, ByteSize testFileSize)
+        private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader)
         {
             var up = uploader.GetName().Replace("<", "").Replace(">", "");
             var down = downloader.GetName().Replace("<", "").Replace(">", "");
-            var label = $"FROM{up}TO{down}";
+            var label = $"~from:{up}-to:{down}~";
             return test.GenerateTestFile(testFileSize, label);
         }
     }

DistTestCore/Marketplace/CodexContractsContainerRecipe.cs

@@ -4,15 +4,11 @@ namespace DistTestCore.Marketplace
 {
     public class CodexContractsContainerRecipe : ContainerRecipeFactory
     {
-        #if Arm64
-            public const string DockerImage = "emizzle/codex-contracts-deployment:latest";
-        #else
-            public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint2";
-        #endif
-        public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json";
-        public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json";
+        public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
+        public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
 
-        protected override string Image => DockerImage;
+        public override string AppName => "codex-contracts";
+        public override string Image => "codexstorage/codex-contracts-eth:latest-dist-tests";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -6,8 +6,8 @@ namespace DistTestCore.Marketplace
     public class CodexContractsStarter : BaseStarter
     {
 
-        public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public CodexContractsStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
@@ -15,7 +15,7 @@ namespace DistTestCore.Marketplace
         {
             LogStart("Deploying Codex Marketplace...");
 
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]);
 
             var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig);
@@ -91,8 +91,13 @@ namespace DistTestCore.Marketplace
         {
             debug(line);
             if (line.Contains(RequiredCompiledString)) SeenCompileString = true;
+            if (line.Contains(ReadyString))
+            {
+                if (!SeenCompileString) throw new Exception("CodexContracts deployment failed. " +
+                    "Solidity files not compiled before process exited.");
 
-            if (SeenCompileString && line.Contains(ReadyString)) Found = true;
+                Found = true;
+            }
         }
     }
 }

DistTestCore/Marketplace/GethBootstrapNodeStarter.cs

@@ -4,8 +4,8 @@ namespace DistTestCore.Marketplace
 {
     public class GethBootstrapNodeStarter : BaseStarter
     {
-        public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public GethBootstrapNodeStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
@@ -14,7 +14,7 @@ namespace DistTestCore.Marketplace
             LogStart("Starting Geth bootstrap node...");
             var startupConfig = CreateBootstrapStartupConfig();
             
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
             var bootstrapContainer = containers.Containers[0];

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -7,8 +7,8 @@ namespace DistTestCore.Marketplace
     {
         private int companionAccountIndex = 0;
 
-        public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public GethCompanionNodeStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
@@ -18,7 +18,7 @@ namespace DistTestCore.Marketplace
 
             var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
 
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
             var container = containers.Containers[0];

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -4,19 +4,14 @@ namespace DistTestCore.Marketplace
 {
     public class GethContainerRecipe : ContainerRecipeFactory
     {
-        #if Arm64
-            public const string DockerImage = "emizzle/geth-confenv:latest";
-        #else
-            public const string DockerImage = "thatbenbierens/geth-confenv:onethousand";
-        #endif
+        private const string defaultArgs = "--ipcdisable --syncmode full";
 
         public const string HttpPortTag = "http_port";
         public const string DiscoveryPortTag = "disc_port";
-        private const string defaultArgs = "--ipcdisable --syncmode full";
-
         public const string AccountsFilename = "accounts.csv";
 
-        protected override string Image => DockerImage;
+        public override string AppName => "geth";
+        public override string Image => "codexstorage/dist-tests-geth:latest";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -1,8 +1,11 @@
 using DistTestCore.Codex;
 using DistTestCore.Helpers;
+using Logging;
+using Newtonsoft.Json;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
 using System.Numerics;
+using System.Linq;
 using Utils;
 
 namespace DistTestCore.Marketplace
@@ -10,7 +13,7 @@ namespace DistTestCore.Marketplace
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration);
-        string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
+        StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, DateTime? expiry = null, uint? tolerance = null);
         void AssertThatBalance(IResolveConstraint constraint, string message = "");
         TestToken GetBalance();
     }
@@ -30,17 +33,18 @@ namespace DistTestCore.Marketplace
             this.codexAccess = codexAccess;
         }
 
-        public string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, DateTime? expiry = null, uint? tolerance = null)
         {
+            var expiryTimestamp = expiry is null ? null : ((DateTimeOffset)expiry).ToUnixTimeSeconds().ToString();
             var request = new CodexSalesRequestStorageRequest
             {
                 duration = ToDecInt(duration.TotalSeconds),
                 proofProbability = ToDecInt(proofProbability),
                 reward = ToDecInt(pricePerSlotPerSecond),
                 collateral = ToDecInt(requiredCollateral),
-                expiry = null,
+                expiry = expiryTimestamp,
                 nodes = minRequiredNumberOfNodes,
-                tolerance = null,
+                tolerance = tolerance,
             };
 
             Log($"Requesting storage for: {contentId.Id}... (" +
@@ -48,7 +52,9 @@ namespace DistTestCore.Marketplace
                 $"requiredCollateral: {requiredCollateral}, " +
                 $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
                 $"proofProbability: {proofProbability}, " +
-                $"duration: {Time.FormatDuration(duration)})");
+                $"duration: {Time.FormatDuration(duration)}, " +
+                $"expiry: {expiry:yyyy-MM-dd HH:mm:ss}, " +
+                $"tolerance: {tolerance})");
 
             var response = codexAccess.RequestStorage(request, contentId.Id);
 
@@ -57,9 +63,9 @@ namespace DistTestCore.Marketplace
                 throw new InvalidOperationException(response);
             }
 
-            Log($"Storage requested successfully. PurchaseId: {response}");
+            Log($"Storage requested successfully. PurchaseId: '{response}'.");
 
-            return response;
+            return new StoragePurchaseContract(lifecycle.Log, codexAccess, response, duration);
         }
 
         public string MakeStorageAvailable(ByteSize totalSpace, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration)
@@ -121,10 +127,10 @@ namespace DistTestCore.Marketplace
 
     public class MarketplaceUnavailable : IMarketplaceAccess
     {
-        public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, DateTime? expiry = null, uint? tolerance = null)
         {
             Unavailable();
-            return string.Empty;
+            return null!;
         }
 
         public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration)
@@ -150,4 +156,121 @@ namespace DistTestCore.Marketplace
             throw new InvalidOperationException();
         }
     }
+
+    public class StoragePurchaseContract
+    {
+        private readonly BaseLog log;
+        private readonly CodexAccess codexAccess;
+        private DateTime? contractStartUtc;
+
+        public StoragePurchaseContract(BaseLog log, CodexAccess codexAccess, string purchaseId, TimeSpan contractDuration)
+        {
+            this.log = log;
+            this.codexAccess = codexAccess;
+            PurchaseId = purchaseId;
+            ContractDuration = contractDuration;
+        }
+
+        public string PurchaseId { get; }
+        public TimeSpan ContractDuration { get; }
+
+        public void WaitForStorageContractStarted()
+        {
+            WaitForStorageContractStarted(TimeSpan.FromSeconds(30));
+        }
+
+        public void WaitForStorageContractFinished()
+        {
+            if (!contractStartUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var gracePeriod = TimeSpan.FromSeconds(10);
+            var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
+            var timeout = (ContractDuration - currentContractTime) + gracePeriod;
+            WaitForStorageContractState(timeout, "finished");
+        }
+
+        /// <summary>
+        /// Wait for contract to terminate. Which means that it reaches one of the following states: Finished, Cancelled, Failed
+        /// </summary>
+        public void WaitForStorageContractTerminated()
+        {
+            if (!contractStartUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var gracePeriod = TimeSpan.FromSeconds(10);
+            var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
+            var timeout = (ContractDuration - currentContractTime) + gracePeriod;
+            string[] terminatedStates = { "finished", "failed", "cancelled" };
+            WaitForStorageContractState(timeout, terminatedStates);
+        }
+
+        public void WaitForStorageContractFailed(TimeSpan timeout)
+        {
+            WaitForStorageContractState(timeout, "failed");
+            contractStartUtc = DateTime.UtcNow;
+        }
+
+
+        /// <summary>
+        /// Wait for contract to start. Max timeout depends on contract filesize. Allows more time for larger files.
+        /// </summary>
+        public void WaitForStorageContractStarted(ByteSize contractFileSize)
+        {
+            var filesizeInMb = contractFileSize.SizeInBytes / (1024 * 1024);
+            var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0);
+
+            WaitForStorageContractStarted(maxWaitTime);
+        }
+
+        public void WaitForStorageContractStarted(TimeSpan timeout)
+        {
+            WaitForStorageContractState(timeout, "started");
+            contractStartUtc = DateTime.UtcNow;
+        }
+
+        private void WaitForStorageContractState(TimeSpan timeout, string desiredState)
+        {
+            string[] states = { desiredState };
+            WaitForStorageContractState(timeout, states);
+        }
+
+        private void WaitForStorageContractState(TimeSpan timeout, string[] desiredState)
+        {
+            var lastState = "";
+            var waitStart = DateTime.UtcNow;
+
+            log.Log($"Waiting for {Time.FormatDuration(timeout)} for contract '{PurchaseId}' to reach state '{string.Join("/", desiredState)}'.");
+            while (!desiredState.Contains(lastState))
+            {
+                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
+                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
+                if (purchaseStatus != null && purchaseStatus.state != lastState)
+                {
+                    lastState = purchaseStatus.state;
+                    log.Debug("Purchase status: " + statusJson);
+                }
+
+                Thread.Sleep(1000);
+
+                if (lastState == "errored")
+                {
+                    Assert.Fail("Contract errored: " + statusJson);
+                }
+
+                if (DateTime.UtcNow - waitStart > timeout)
+                {
+                    Assert.Fail($"Contract did not reach '{string.Join("/", desiredState)}' within timeout. {statusJson}");
+                }
+            }
+            log.Log($"Contract '{string.Join("/", desiredState)}'.");
+        }
+
+        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
+        {
+            return codexAccess.GetPurchaseStatus(purchaseId);
+        }
+    }
 }

DistTestCore/Metrics/PrometheusContainerRecipe.cs

@@ -4,9 +4,8 @@ namespace DistTestCore.Metrics
 {
     public class PrometheusContainerRecipe : ContainerRecipeFactory
     {
-        public const string DockerImage = "thatbenbierens/prometheus-envconf:latest";
-
-        protected override string Image => DockerImage;
+        public override string AppName => "prometheus";
+        public override string Image => "codexstorage/dist-tests-prometheus:latest";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

DistTestCore/OnlineCodexNode.cs

@@ -18,6 +18,7 @@ namespace DistTestCore
         IDownloadedLog DownloadLog();
         IMetricsAccess Metrics { get; }
         IMarketplaceAccess Marketplace { get; }
+        CodexDebugVersionResponse Version { get; }
         ICodexSetup BringOffline();
     }
 
@@ -34,12 +35,14 @@ namespace DistTestCore
             Group = group;
             Metrics = metricsAccess;
             Marketplace = marketplaceAccess;
+            Version = new CodexDebugVersionResponse();
         }
 
         public CodexAccess CodexAccess { get; }
         public CodexNodeGroup Group { get; }
         public IMetricsAccess Metrics { get; }
         public IMarketplaceAccess Marketplace { get; }
+        public CodexDebugVersionResponse Version { get; private set; }
 
         public string GetName()
         {
@@ -72,9 +75,6 @@ namespace DistTestCore
             if (string.IsNullOrEmpty(response)) Assert.Fail("Received empty response.");
             if (response.StartsWith(UploadFailedMessage)) Assert.Fail("Node failed to store block.");
 
-            var logReplacement = $"(CID:{file.Describe()})";
-            Log($"ContentId '{response}' is {logReplacement}");
-            lifecycle.Log.AddStringReplace(response, logReplacement);
             Log($"Uploaded file. Received contentId: '{response}'.");
             return new ContentId(response);
         }
@@ -114,19 +114,30 @@ namespace DistTestCore
             return Group.BringOffline();
         }
 
+        public void EnsureOnlineGetVersionResponse()
+        {
+            var debugInfo = CodexAccess.GetDebugInfo();
+            var nodePeerId = debugInfo.id;
+            var nodeName = CodexAccess.Container.Name;
+
+            if (!debugInfo.codex.IsValid())
+            {
+                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}");
+            }
+
+            lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
+            lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
+            Version = debugInfo.codex;
+        }
+
         private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
         {
             var multiAddress = peerInfo.addrs.First();
             // Todo: Is there a case where First address in list is not the way?
 
-            if (Group == peer.Group)
-            {
-                return multiAddress;
-            }
-
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip);
+            return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip);
         }
 
         private void DownloadToFile(string contentId, TestFile file)

DistTestCore/PrometheusStarter.cs

@@ -7,18 +7,18 @@ namespace DistTestCore
 {
     public class PrometheusStarter : BaseStarter
     {
-        public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public PrometheusStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
-        public RunningContainers CollectMetricsFor(RunningContainers containers)
+        public RunningContainers CollectMetricsFor(RunningContainers[] containers)
         {
             LogStart($"Starting metrics server for {containers.Describe()}");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers)));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers())));
 
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 

DistTestCore/TestLifecycle.cs

@@ -1,4 +1,7 @@
-using DistTestCore.Logs;
+using DistTestCore.Codex;
+using DistTestCore.Logs;
+using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
 using KubernetesWorkflow;
 using Logging;
 using Utils;
@@ -7,33 +10,36 @@ namespace DistTestCore
 {
     public class TestLifecycle
     {
-        private DateTime testStart = DateTime.MinValue;
+        private readonly DateTime testStart;
 
-        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet)
-            : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)))
-        {
-        }
-
-        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
+        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testsType, string testNamespace)
         {
             Log = log;
             Configuration = configuration;
             TimeSet = timeSet;
 
+            var podLabels = new PodLabels(testsType, GetApplicationIds());
+            WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), podLabels, testNamespace);
+
             FileManager = new FileManager(Log, configuration);
-            CodexStarter = new CodexStarter(this, workflowCreator);
-            PrometheusStarter = new PrometheusStarter(this, workflowCreator);
-            GethStarter = new GethStarter(this, workflowCreator);
+            CodexStarter = new CodexStarter(this);
+            PrometheusStarter = new PrometheusStarter(this);
+            GethStarter = new GethStarter(this);
             testStart = DateTime.UtcNow;
+            CodexVersion = null;
+
+            Log.WriteLogTag();
         }
 
         public BaseLog Log { get; }
         public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
+        public WorkflowCreator WorkflowCreator { get; }
         public FileManager FileManager { get; }
         public CodexStarter CodexStarter { get; }
         public PrometheusStarter PrometheusStarter { get; }
         public GethStarter GethStarter { get; }
+        public CodexDebugVersionResponse? CodexVersion { get; private set; }
 
         public void DeleteAllResources()
         {
@@ -58,5 +64,28 @@ namespace DistTestCore
             var testDuration = DateTime.UtcNow - testStart;
             return Time.FormatDuration(testDuration);
         }
+
+        public void SetCodexVersion(CodexDebugVersionResponse version)
+        {
+            if (CodexVersion == null) CodexVersion = version;
+        }
+
+        public ApplicationIds GetApplicationIds()
+        {
+            return new ApplicationIds(
+                codexId: GetCodexId(),
+                gethId: new GethContainerRecipe().Image,
+                prometheusId: new PrometheusContainerRecipe().Image,
+                codexContractsId: new CodexContractsContainerRecipe().Image
+            );
+        }
+
+        private string GetCodexId()
+        {
+            var v = CodexVersion;
+            if (v == null) return new CodexContainerRecipe().Image;
+            if (v.version != "untagged build") return v.version;
+            return v.revision;
+        }
     }
 }

DistTestCore/Tokens.cs

@@ -58,6 +58,16 @@
         {
             return $"{Amount} TestTokens";
         }
+
+        public static TestToken operator - (TestToken t1, int amount)
+        {
+            return t1 - Convert.ToDecimal(amount);
+        }
+
+        public static TestToken operator - (TestToken t1, decimal amount)
+        {
+            return (t1.Amount - amount).TestTokens();
+        }
     }
 
     public static class TokensIntExtensions

KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -27,7 +27,8 @@
             return recipe;
         }
 
-        protected abstract string Image { get; }
+        public abstract string AppName { get; }
+        public abstract string Image { get; }
         protected int ContainerNumber { get; private set; } = 0;
         protected int Index { get; private set; } = 0;
         protected abstract void Initialize(StartupConfig config);

KubernetesWorkflow/K8sController.cs

@@ -11,14 +11,16 @@ namespace KubernetesWorkflow
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownPods;
         private readonly WorkflowNumberSource workflowNumberSource;
+        private readonly PodLabels podLabels;
         private readonly K8sClient client;
 
-        public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
+        public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace, PodLabels podLabels)
         {
             this.log = log;
             this.cluster = cluster;
             this.knownPods = knownPods;
             this.workflowNumberSource = workflowNumberSource;
+            this.podLabels = podLabels;
             client = new K8sClient(cluster.GetK8sClientConfig());
 
             K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
@@ -362,7 +364,7 @@ namespace KubernetesWorkflow
 
         private IDictionary<string, string> GetSelector()
         {
-            return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
+            return podLabels.GetLabels();
         }
 
         private IDictionary<string, string> GetRunnerNamespaceSelector()

KubernetesWorkflow/PodLabels.cs

@@ -0,0 +1,52 @@
+using Logging;
+
+namespace KubernetesWorkflow
+{
+    public class PodLabels
+    {
+        private readonly Dictionary<string, string> labels = new Dictionary<string, string>();
+
+        private PodLabels(PodLabels source)
+        {
+            labels = source.labels.ToDictionary(p => p.Key, p => p.Value);
+        }
+
+        public PodLabels(string testsType, ApplicationIds applicationIds)
+        {
+            Add("tests-type", testsType);
+            Add("runid", NameUtils.GetRunId());
+            Add("testid", NameUtils.GetTestId());
+            Add("category", NameUtils.GetCategoryName());
+            Add("fixturename", NameUtils.GetRawFixtureName());
+            Add("testname", NameUtils.GetTestMethodName());
+
+            if (applicationIds == null) return;
+            Add("codexid", applicationIds.CodexId);
+            Add("gethid", applicationIds.GethId);
+            Add("prometheusid", applicationIds.PrometheusId);
+            Add("codexcontractsid", applicationIds.CodexContractsId);
+        }
+
+        public PodLabels GetLabelsForAppName(string appName)
+        {
+            var pl = new PodLabels(this);
+            pl.Add("app", appName);
+            return pl;
+        }
+
+        private void Add(string key, string value)
+        {
+            labels.Add(key, 
+                value.ToLowerInvariant()
+                .Replace(":","-")
+                .Replace("/", "-")
+                .Replace("\\", "-")
+            );
+        }
+
+        internal Dictionary<string, string> GetLabels()
+        {
+            return labels;
+        }
+    }
+}

KubernetesWorkflow/RunningContainers.cs

@@ -40,4 +40,17 @@ namespace KubernetesWorkflow
         public Address ClusterExternalAddress { get; }
         public Address ClusterInternalAddress { get; }
     }
+
+    public static class RunningContainersExtensions
+    {
+        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
+        {
+            return runningContainers.SelectMany(c => c.Containers).ToArray();
+        }
+
+        public static string Describe(this RunningContainers[] runningContainers)
+        {
+            return string.Join(",", runningContainers.Select(c => c.Describe()));
+        }
+    }
 }

KubernetesWorkflow/StartupWorkflow.cs

@@ -10,19 +10,23 @@ namespace KubernetesWorkflow
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownK8SPods;
         private readonly string testNamespace;
+        private readonly PodLabels podLabels;
         private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
 
-        internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
+        internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace, PodLabels podLabels)
         {
             this.log = log;
             this.numberSource = numberSource;
             this.cluster = cluster;
             this.knownK8SPods = knownK8SPods;
             this.testNamespace = testNamespace;
+            this.podLabels = podLabels;
         }
 
         public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
+            var pl = podLabels.GetLabelsForAppName(recipeFactory.AppName);
+
             return K8s(controller =>
             {
                 var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
@@ -30,7 +34,7 @@ namespace KubernetesWorkflow
                 var runningPod = controller.BringOnline(recipes, location);
 
                 return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig));
-            });
+            }, pl);
         }
 
         public void Stop(RunningContainers runningContainers)
@@ -147,14 +151,22 @@ namespace KubernetesWorkflow
 
         private void K8s(Action<K8sController> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels);
             action(controller);
             controller.Dispose();
         }
 
         private T K8s<T>(Func<K8sController, T> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels);
+            var result = action(controller);
+            controller.Dispose();
+            return result;
+        }
+
+        private T K8s<T>(Func<K8sController, T> action, PodLabels labels)
+        {
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, labels);
             var result = action(controller);
             controller.Dispose();
             return result;

KubernetesWorkflow/WorkflowCreator.cs

@@ -10,18 +10,15 @@ namespace KubernetesWorkflow
         private readonly KnownK8sPods knownPods = new KnownK8sPods();
         private readonly K8sCluster cluster;
         private readonly BaseLog log;
+        private readonly PodLabels podLabels;
         private readonly string testNamespace;
 
-        public WorkflowCreator(BaseLog log, Configuration configuration)
-            : this(log, configuration, Guid.NewGuid().ToString().ToLowerInvariant())
-        {
-        }
-
-        public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespacePostfix)
+        public WorkflowCreator(BaseLog log, Configuration configuration, PodLabels podLabels, string testNamespace)
         {
             cluster = new K8sCluster(configuration);
             this.log = log;
-            testNamespace = testNamespacePostfix;
+            this.podLabels = podLabels;
+            this.testNamespace = testNamespace.ToLowerInvariant();
         }
 
         public StartupWorkflow CreateWorkflow()
@@ -29,7 +26,7 @@ namespace KubernetesWorkflow
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
                                                                     containerNumberSource);
 
-            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);
+            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace, podLabels);
         }
     }
 }

Logging/ApplicationIds.cs

@@ -0,0 +1,18 @@
+namespace Logging
+{
+    public class ApplicationIds
+    {
+        public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId)
+        {
+            CodexId = codexId;
+            GethId = gethId;
+            PrometheusId = prometheusId;
+            CodexContractsId = codexContractsId;
+        }
+
+        public string CodexId { get; }
+        public string GethId { get; }
+        public string PrometheusId { get; }
+        public string CodexContractsId { get; }
+    }
+}

Logging/BaseLog.cs

@@ -73,6 +73,14 @@ namespace Logging
             return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
         }
 
+        public void WriteLogTag()
+        {
+            var runId = NameUtils.GetRunId();
+            var category = NameUtils.GetCategoryName();
+            var name = NameUtils.GetTestMethodName();
+            LogFile.WriteRaw($"{runId} {category} {name}");
+        }
+
         private string ApplyReplacements(string str)
         {
             foreach (var replacement in replacements)

Logging/FixtureLog.cs

@@ -1,20 +1,14 @@
-using NUnit.Framework;
-
-namespace Logging
+namespace Logging
 {
     public class FixtureLog : BaseLog
     {
-        private readonly DateTime start;
         private readonly string fullName;
         private readonly LogConfig config;
 
-        public FixtureLog(LogConfig config, string name = "")
+        public FixtureLog(LogConfig config, DateTime start, string name = "")
             : base(config.DebugEnabled)
         {
-            start = DateTime.UtcNow;
-            var folder = DetermineFolder(config);
-            var fixtureName = GetFixtureName(name);
-            fullName = Path.Combine(folder, fixtureName);
+            fullName = NameUtils.GetFixtureFullName(config, start, name);
             this.config = config;
         }
 
@@ -32,28 +26,5 @@ namespace Logging
         {
             return fullName;
         }
-
-        private string DetermineFolder(LogConfig config)
-        {
-            return Path.Join(
-               config.LogRoot,
-               $"{start.Year}-{Pad(start.Month)}",
-               Pad(start.Day));
-        }
-
-        private string GetFixtureName(string name)
-        {
-            var test = TestContext.CurrentContext.Test;
-            var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
-            if (!string.IsNullOrEmpty(name)) className = name;
-
-            return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
-        }
-
-        private static string Pad(int n)
-        {
-            return n.ToString().PadLeft(2, '0');
-        }
-
     }
 }

Logging/LogFile.cs

@@ -49,7 +49,7 @@
 
         private static string GetTimestamp()
         {
-            return $"[{DateTime.UtcNow.ToString("u")}]";
+            return $"[{DateTime.UtcNow.ToString("o")}]";
         }
 
         private void EnsurePathExists(string filename)

Logging/NameUtils.cs

@@ -0,0 +1,85 @@
+using NUnit.Framework;
+
+namespace Logging
+{
+    public static class NameUtils
+    {
+        public static string GetTestMethodName(string name = "")
+        {
+            if (!string.IsNullOrEmpty(name)) return name;
+            var test = TestContext.CurrentContext.Test;
+            var args = FormatArguments(test);
+            return ReplaceInvalidCharacters($"{test.MethodName}{args}");
+        }
+
+        public static string GetFixtureFullName(LogConfig config, DateTime start, string name)
+        {
+            var folder = DetermineFolder(config, start);
+            var fixtureName = GetFixtureName(name, start);
+            return Path.Combine(folder, fixtureName);
+        }
+
+        public static string GetRawFixtureName()
+        {
+            var test = TestContext.CurrentContext.Test;
+            if (test.ClassName!.Contains("AdhocContext")) return "none";
+            var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
+            return className.Replace('.', '-');
+        }
+
+        public static string GetCategoryName()
+        {
+            var test = TestContext.CurrentContext.Test;
+            if (test.ClassName!.Contains("AdhocContext")) return "none";
+            return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.'));
+        }
+
+        public static string GetTestId()
+        {
+            return GetEnvVar("TESTID");
+        }
+
+        public static string GetRunId()
+        {
+            return GetEnvVar("RUNID");
+        }
+
+        private static string GetEnvVar(string name)
+        {
+            var v = Environment.GetEnvironmentVariable(name);
+            if (string.IsNullOrEmpty(v)) return $"EnvVar-{name}-NotSet";
+            return v;
+        }
+
+        private static string FormatArguments(TestContext.TestAdapter test)
+        {
+            if (test.Arguments == null || !test.Arguments.Any()) return "";
+            return $"[{string.Join(',', test.Arguments)}]";
+        }
+
+        private static string ReplaceInvalidCharacters(string name)
+        {
+            return name.Replace(":", "_");
+        }
+
+        private static string DetermineFolder(LogConfig config, DateTime start)
+        {
+            return Path.Join(
+               config.LogRoot,
+               $"{start.Year}-{Pad(start.Month)}",
+               Pad(start.Day));
+        }
+
+        private static string GetFixtureName(string name, DateTime start)
+        {
+            var className = GetRawFixtureName();
+            if (!string.IsNullOrEmpty(name)) className = name;
+            return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
+        }
+
+        private static string Pad(int n)
+        {
+            return n.ToString().PadLeft(2, '0');
+        }
+    }
+}

Logging/StatusLog.cs

@@ -0,0 +1,67 @@
+using Newtonsoft.Json;
+
+namespace Logging
+{
+    public class StatusLog
+    {
+        private readonly object fileLock = new object();
+        private readonly string fullName;
+        private readonly string fixtureName;
+
+        public StatusLog(LogConfig config, DateTime start, string name = "")
+        {
+            fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log";
+            fixtureName = NameUtils.GetRawFixtureName();
+        }
+
+        public void ConcludeTest(string resultStatus, string testDuration, ApplicationIds applicationIds)
+        {
+            Write(new StatusLogJson
+            {
+                @timestamp = DateTime.UtcNow.ToString("o"),
+                runid = NameUtils.GetRunId(),
+                status = resultStatus,
+                testid = NameUtils.GetTestId(),
+                codexid = applicationIds.CodexId,
+                gethid = applicationIds.GethId,
+                prometheusid = applicationIds.PrometheusId,
+                codexcontractsid = applicationIds.CodexContractsId,
+                category = NameUtils.GetCategoryName(),
+                fixturename = fixtureName,
+                testname = NameUtils.GetTestMethodName(),
+                testduration = testDuration
+            });
+        }
+
+        private void Write(StatusLogJson json)
+        {
+            try
+            {
+                lock (fileLock)
+                {
+                    File.AppendAllLines(fullName, new[] { JsonConvert.SerializeObject(json) });
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine("Unable to write to status log: " + ex);
+            }
+        }
+    }
+
+    public class StatusLogJson
+    {
+        public string @timestamp { get; set; } = string.Empty;
+        public string runid { get; set; } = string.Empty;
+        public string status { get; set; } = string.Empty;
+        public string testid { get; set; } = string.Empty; 
+        public string codexid { get; set; } = string.Empty;
+        public string gethid { get; set; } = string.Empty;
+        public string prometheusid { get; set; } = string.Empty;
+        public string codexcontractsid { get; set; } = string.Empty;
+        public string category { get; set; } = string.Empty;
+        public string fixturename { get; set; } = string.Empty;
+        public string testname { get; set; } = string.Empty;
+        public string testduration { get; set;} = string.Empty;
+    }
+}

Logging/TestLog.cs

@@ -10,7 +10,7 @@ namespace Logging
         public TestLog(string folder, bool debug, string name = "")
             : base(debug)
         {
-            methodName = GetMethodName(name);
+            methodName = NameUtils.GetTestMethodName(name);
             fullName = Path.Combine(folder, methodName);
 
             Log($"*** Begin: {methodName}");
@@ -37,24 +37,5 @@ namespace Logging
         {
             return fullName;
         }
-
-        private string GetMethodName(string name)
-        {
-            if (!string.IsNullOrEmpty(name)) return name;
-            var test = TestContext.CurrentContext.Test;
-            var args = FormatArguments(test);
-            return ReplaceInvalidCharacters($"{test.MethodName}{args}");
-        }
-
-        private static string FormatArguments(TestContext.TestAdapter test)
-        {
-            if (test.Arguments == null || !test.Arguments.Any()) return "";
-            return $"[{string.Join(',', test.Arguments)}]";
-        }
-
-        private static string ReplaceInvalidCharacters(string name)
-        {
-            return name.Replace(":", "_");
-        }
     }
 }

Tests/BasicTests/ExampleTests.cs

@@ -1,6 +1,6 @@
 using DistTestCore;
+using DistTestCore.Codex;
 using NUnit.Framework;
-using Utils;
 
 namespace Tests.BasicTests
 {
@@ -44,8 +44,10 @@ namespace Tests.BasicTests
         {
             var sellerInitialBalance = 234.TestTokens();
             var buyerInitialBalance = 1000.TestTokens();
+            var fileSize = 10.MB();
 
             var seller = SetupCodexNode(s => s
+                            .WithLogLevel(CodexLogLevel.Trace, "marketplace", "sales", "proving", "reservations")
                             .WithStorageQuota(11.GB())
                             .EnableMarketplace(sellerInitialBalance));
 
@@ -56,25 +58,28 @@ namespace Tests.BasicTests
                 maxCollateral: 20.TestTokens(),
                 maxDuration: TimeSpan.FromMinutes(3));
 
-            var testFile = GenerateTestFile(10.MB());
+            var testFile = GenerateTestFile(fileSize);
 
             var buyer = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, "marketplace", "purchasing", "node", "restapi")
                 .WithBootstrapNode(seller)
                 .EnableMarketplace(buyerInitialBalance));
-            
+
+            buyer.Marketplace.AssertThatBalance(Is.EqualTo(buyerInitialBalance));
+
             var contentId = buyer.UploadFile(testFile);
-            buyer.Marketplace.RequestStorage(contentId,
+            var purchaseContract = buyer.Marketplace.RequestStorage(contentId,
                 pricePerSlotPerSecond: 2.TestTokens(),
                 requiredCollateral: 10.TestTokens(),
                 minRequiredNumberOfNodes: 1,
                 proofProbability: 5,
                 duration: TimeSpan.FromMinutes(1));
 
-            Time.Sleep(TimeSpan.FromSeconds(10));
+            purchaseContract.WaitForStorageContractStarted(fileSize);
 
             seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
 
-            Time.Sleep(TimeSpan.FromMinutes(1));
+            purchaseContract.WaitForStorageContractFinished();
 
             seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
             buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");

Tests/Marketplace/Interactions.cs

@@ -0,0 +1,87 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.BasicTests
+{
+    [TestFixture]
+    public class SlotSelection : AutoBootstrapDistTest
+    {
+
+        [Test]
+        public void RequestExpiresIfNotFilledAndMoneyAreReturned()
+        {
+            var sellerInitialBalance = 234.TestTokens();
+            var buyerInitialBalance = 1000.TestTokens();
+            var fileSize = 10.MB();
+
+            var seller1 = SetupCodexNode(s => s
+                            .WithLogLevel(CodexLogLevel.Trace, "marketplace", "sales", "proving", "reservations")
+                            .WithStorageQuota(50.MB())
+                            .EnableMarketplace(sellerInitialBalance)
+                            .WithName("seller1"));
+
+            seller1.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance));
+
+            // The two availabilities are needed until https://github.com/codex-storage/nim-codex/pull/535 is merged
+            seller1.Marketplace.MakeStorageAvailable(
+                size: 11.MB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+            seller1.Marketplace.MakeStorageAvailable(
+                size: 11.MB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            var seller2 = SetupCodexNode(s => s
+                            .WithLogLevel(CodexLogLevel.Trace, "marketplace", "sales", "proving", "reservations")
+                            .WithStorageQuota(50.MB())
+                            .EnableMarketplace(sellerInitialBalance)
+                            .WithName("seller2"));
+
+            seller2.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance));
+            seller2.Marketplace.MakeStorageAvailable(
+                size: 11.MB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+            seller2.Marketplace.MakeStorageAvailable(
+                size: 11.MB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            var testFile = GenerateTestFile(fileSize);
+            var buyer = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, "marketplace", "purchasing", "node", "restapi")
+                .EnableMarketplace(buyerInitialBalance)
+                .WithName("buyer"));
+
+            buyer.Marketplace.AssertThatBalance(Is.EqualTo(buyerInitialBalance));
+
+            var contentId = buyer.UploadFile(testFile);
+            var purchaseContract = buyer.Marketplace.RequestStorage(contentId,
+                pricePerSlotPerSecond: 2.TestTokens(),
+                requiredCollateral: 10.TestTokens(),
+                minRequiredNumberOfNodes: 3,
+                proofProbability: 5,
+                duration: TimeSpan.FromMinutes(1),
+                expiry: DateTime.Now.AddMinutes(2));
+
+            Time.Sleep(TimeSpan.FromSeconds(100));
+
+            seller1.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
+            seller2.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
+            buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");
+
+            purchaseContract.WaitForStorageContractFailed(TimeSpan.FromSeconds(120));
+
+            seller1.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance), "Seller was not returned collateral.");
+            seller2.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance), "Seller was not returned collateral.");
+            buyer.Marketplace.AssertThatBalance(Is.EqualTo(buyerInitialBalance), "Buyer was not returned money for the request.");
+        }
+    }
+}

Tests/Marketplace/Proofs.cs

@@ -0,0 +1,97 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.BasicTests
+{
+    [TestFixture]
+    public class MarketplaceTests : DistTest
+    {
+        [Test]
+        public void HostThatMissesProofsIsPaidOutLessThanHostThatDoesNotMissProofs()
+        {
+            var sellerInitialBalance = 234.TestTokens();
+            var buyerInitialBalance = 1000.TestTokens();
+
+            var seller = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, "marketplace", "sales", "proving", "reservations")
+                .WithStorageQuota(21.GB())
+                .EnableMarketplace(sellerInitialBalance)
+                .WithName("seller"));
+
+            var sellerWithFailures = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, "marketplace", "sales", "proving", "reservations")
+                .WithStorageQuota(21.GB())
+                .WithBootstrapNode(seller)
+                .WithSimulateProofFailures(2)
+                .EnableMarketplace(sellerInitialBalance)
+                .WithName("seller with failures"));
+
+            var buyer = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, "marketplace", "purchasing", "node", "restapi")
+                .WithBootstrapNode(seller)
+                .EnableMarketplace(buyerInitialBalance)
+                .WithName("buyer"));
+
+            var validator = SetupCodexNode(s => s
+                .WithLogLevel(CodexLogLevel.Trace, "validator")
+                .WithBootstrapNode(seller)
+                // .WithValidator()
+                .EnableMarketplace(0.TestTokens(), 2.Eth(), true)
+                .WithName("validator"));
+
+            seller.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance));
+            sellerWithFailures.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance));
+            buyer.Marketplace.AssertThatBalance(Is.EqualTo(buyerInitialBalance));
+
+            seller.Marketplace.MakeStorageAvailable(
+                size: 10.GB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            sellerWithFailures.Marketplace.MakeStorageAvailable(
+                size: 10.GB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            seller.Marketplace.MakeStorageAvailable(
+                size: 10.GB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            sellerWithFailures.Marketplace.MakeStorageAvailable(
+                size: 10.GB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 20.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(3));
+
+            var fileSize = 10.MB();
+            var testFile = GenerateTestFile(fileSize);
+            var contentId = buyer.UploadFile(testFile);
+
+            var purchaseContract = buyer.Marketplace.RequestStorage(
+                contentId,
+                pricePerSlotPerSecond: 2.TestTokens(),
+                requiredCollateral: 10.TestTokens(),
+                minRequiredNumberOfNodes: 2,
+                proofProbability: 2,
+                duration: TimeSpan.FromMinutes(3));
+
+            // Time.Sleep(TimeSpan.FromMinutes(1));
+
+            // seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
+
+            purchaseContract.WaitForStorageContractStarted(fileSize);
+            purchaseContract.WaitForStorageContractFinished();
+
+            var sellerBalance = seller.Marketplace.GetBalance();
+            sellerWithFailures.Marketplace.AssertThatBalance(Is.LessThan(sellerBalance), "Seller that was slashed should have less balance than seller that was not slashed.");
+
+            new List<IOnlineCodexNode>(){seller, sellerWithFailures, buyer, validator}.ForEach(node => node.DownloadLog());
+        }
+    }
+}

docker/docker-compose.yaml

@@ -0,0 +1,11 @@
+services:
+  dist-test-run:
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile
+    environment:
+      - CODEXDOCKERIMAGE=codexstorage/nim-codex:sha-14c5270
+      - BRANCH="feature/docker-image-testruns"
+      - KUBECONFIG=/opt/kubeconfig
+      - LOGPATH=/opt/logs
+      - RUNNERLOCATION=ExternalToCluster

docker/job.yaml

@@ -28,6 +28,10 @@ spec:
           value: ${BRANCH}
         - name: SOURCE
           value: ${SOURCE}
+        - name: RUNID
+          value: ${RUNID}
+        - name: TESTID
+          value: ${TESTID}
         volumeMounts:
         - name: kubeconfig
           mountPath: /opt/kubeconfig.yaml