Sets up tests for symmetric, asymmetric and binary speed checks

* Fix job status check

* Add a delay before get Kubernetes job status

.gitattributes

@@ -0,0 +1,10 @@
+# Set default behavior to automatically normalize line endings.
+* text=auto
+
+# Force bash scripts to always use lf line endings so that if a repo is accessed
+# in Unix via a file share from Windows, the scripts will work.
+*.sh text eol=lf
+
+# Likewise, force cmd and batch scripts to always use crlf
+*.cmd text eol=crlf
+*.bat text eol=crlf

.github/workflows/run-continuous-tests.yaml

@@ -80,7 +80,7 @@ env:
   TESTS_TARGET_DURATION: 2d
   TESTS_FILTER: ""
   TESTS_CLEANUP: true
-  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  JOB_MANIFEST: docker/job-continuous-tests.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
   KUBE_VERSION: v1.28.2
 
@@ -146,7 +146,7 @@ jobs:
         if: false
         run: |
           # Variables
-          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to run a node
           duration=600
           namespace="${{ env.NAMESPACE }}"
           pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")

.github/workflows/run-dist-tests.yaml

@@ -32,7 +32,7 @@ env:
   NAMEPREFIX: d-tests-runner
   NAMESPACE: default
   COMMAND: dotnet test Tests/CodexTests
-  JOB_MANIFEST: docker/dist-tests-job.yaml
+  JOB_MANIFEST: docker/job-dist-tests.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
   KUBE_VERSION: v1.28.2
 

.github/workflows/run-release-tests.yaml

@@ -0,0 +1,122 @@
+name: Run Release Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+      workflow_source:
+        description: Workflow source
+        required: false
+        type: string
+        default: ''
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
+  TEST_TYPE: release-tests
+  NAMEPREFIX: r-tests
+  NAMESPACE: default
+  JOB_MANIFEST: docker/job-release-tests.yaml
+  COMMAND: dotnet test Tests/CodexReleaseTests
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.30.5
+
+
+jobs:
+  run_tests:
+    name: Run Release Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
+          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'${COMMAND}'"')" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job to run tests
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}"
+          echo "Runner pod: $(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}')"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+
+      - name: Show Runner logs
+        run: |
+          # Variables
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          namespace="${{ env.NAMESPACE }}"
+          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath='{.items[0].metadata.name}')
+
+          # Check Pod status
+          WAIT=120
+          SECONDS=0
+          sleep=1
+          while (( SECONDS < WAIT )); do
+            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}')
+            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in the $phase state - Get the logs"; break; } || { echo "Pod $pod is in the $phase state - Retry in $sleep second(s) / $((WAIT - SECONDS))"; }
+            sleep $sleep
+          done
+
+          # Get logs
+          while [[ $(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}') == "Running" ]]; do
+            echo "Show ${pod} logs ..."
+            echo "----"
+            kubectl logs $pod -n $namespace -f || true
+            sleep 1
+          done
+
+      - name: Set workflow status from job status
+        run: |
+          sleep 5
+          job_status=$(kubectl get jobs ${{ env.NAMEPREFIX }} -n ${{ env.NAMESPACE }} -o jsonpath='{.status.conditions[0].type}')
+          echo "Job status: $job_status"
+          if [[ "${job_status}" != "Complete" ]]; then exit 1; fi

Framework/BlockchainUtils/BlockCache.cs

@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockCache
     {

Framework/BlockchainUtils/BlockTimeEntry.cs

@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockTimeEntry
     {

Framework/BlockchainUtils/BlockTimeFinder.cs

@@ -1,6 +1,6 @@
 using Logging;
 
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockTimeFinder
     {
@@ -29,7 +29,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();
-            if (moment <= bounds.Genesis.Utc) return null;
+            if (moment < bounds.Genesis.Utc) return null;
+            if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
             if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
 
             return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
@@ -38,7 +39,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetLowestBlockNumberAfter(DateTime moment)
         {
             bounds.Initialize();
-            if (moment >= bounds.Current.Utc) return null;
+            if (moment > bounds.Current.Utc) return null;
+            if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber;
             if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
 
             return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;

Framework/BlockchainUtils/BlockchainBounds.cs

@@ -1,5 +1,11 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
+    public interface IWeb3Blocks
+    {
+        ulong GetCurrentBlockNumber();
+        DateTime? GetTimestampForBlock(ulong blockNumber);
+    }
+
     public class BlockchainBounds
     {
         private readonly BlockCache cache;

Framework/BlockchainUtils/BlockchainUtils.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+	<ItemGroup>
+		<PackageReference Include="Nethereum.Web3" Version="4.14.0" />
+	</ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/BlockchainUtils/ConversionExtensions.cs

@@ -1,7 +1,7 @@
 using Nethereum.Hex.HexTypes;
 using System.Numerics;
 
-namespace NethereumWorkflow
+namespace BlockchainUtils
 {
     public static class ConversionExtensions
     {

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -3,7 +3,8 @@
     public class GiveRewardsCommand
     {
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
-        public string[] EventsOverview { get; set; } = Array.Empty<string>();
+        public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty<ChainEventMessage>();
+        public string[] Errors { get; set; } = Array.Empty<string>();
 
         public bool HasAny()
         {
@@ -16,4 +17,10 @@
         public ulong RewardId { get; set; }
         public string[] UserAddresses { get; set; } = Array.Empty<string>();
     }
+
+    public class ChainEventMessage
+    {
+        public ulong BlockNumber { get; set; }
+        public string Message { get; set; } = string.Empty;
+    }
 }

Framework/GethConnector/GethConnector.cs

@@ -1,4 +1,6 @@
-using CodexContractsPlugin;
+using BlockchainUtils;
+using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 
@@ -18,18 +20,29 @@ namespace GethConnector
                 return null;
             }
 
+            var gethNode = new CustomGethNode(log, new BlockCache(), GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+
+            var config = GetCodexMarketplaceConfig(gethNode, GethInput.MarketplaceAddress);
+
             var contractsDeployment = new CodexContractsDeployment(
+                config: config,
                 marketplaceAddress: GethInput.MarketplaceAddress,
                 abi: GethInput.ABI,
                 tokenAddress: GethInput.TokenAddress
             );
 
-            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
             var contracts = new CodexContractsAccess(log, gethNode, contractsDeployment);
 
             return new GethConnector(gethNode, contracts);
         }
 
+        private static MarketplaceConfig GetCodexMarketplaceConfig(IGethNode gethNode, string marketplaceAddress)
+        {
+            var func = new ConfigurationFunctionBase();
+            var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
+            return response.ReturnValue1;
+        }
+
         private GethConnector(IGethNode gethNode, ICodexContracts codexContracts)
         {
             GethNode = gethNode;

Framework/KubernetesWorkflow/Configuration.cs

@@ -21,8 +21,5 @@ namespace KubernetesWorkflow
 
         [JsonIgnore]
         public IK8sHooks Hooks { get; set; } = new DoNothingK8sHooks();
-
-        [JsonIgnore]
-        public Func<string?, string?> Replacer { get; set; } = s => s;
     }
 }

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -11,13 +11,11 @@ namespace KubernetesWorkflow
         private readonly string podName;
         private readonly string recipeName;
         private readonly string k8sNamespace;
-        private readonly Func<string?, string?> replacer;
         private CancellationTokenSource cts;
         private Task? worker;
         private Exception? workerException;
 
-        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace,
-            Func<string?, string?> replacer)
+        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
         {
             this.log = log;
             this.config = config;
@@ -25,7 +23,6 @@ namespace KubernetesWorkflow
             this.podName = podName;
             this.recipeName = recipeName;
             this.k8sNamespace = k8sNamespace;
-            this.replacer = replacer;
             cts = new CancellationTokenSource();
         }
 
@@ -94,8 +91,8 @@ namespace KubernetesWorkflow
         private void DownloadCrashedContainerLogs(Kubernetes client)
         {
             using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
-            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
-            handler.Log(stream, replacer);
+            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName, containerName);
+            handler.Log(stream);
         }
     }
 }

Framework/KubernetesWorkflow/DownloadedLog.cs

@@ -6,6 +6,7 @@ namespace KubernetesWorkflow
     {
         string ContainerName { get; }
 
+        void IterateLines(Action<string> action);
         void IterateLines(Action<string> action, params string[] thatContain);
         string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
@@ -25,58 +26,39 @@ namespace KubernetesWorkflow
 
         public string ContainerName { get; }
 
-        public void IterateLines(Action<string> action, params string[] thatContain)
+        public void IterateLines(Action<string> action)
         {
             using var file = File.OpenRead(logFile.FullFilename);
             using var streamReader = new StreamReader(file);
 
             var line = streamReader.ReadLine();
             while (line != null)
+            {
+                action(line);
+                line = streamReader.ReadLine();
+            }
+        }
+
+        public void IterateLines(Action<string> action, params string[] thatContain)
+        {
+            IterateLines(line =>
             {
                 if (thatContain.All(line.Contains))
                 {
                     action(line);
                 }
-                line = streamReader.ReadLine();
-            }
+            });
         }
 
         public string[] GetLinesContaining(string expectedString)
         {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-            var lines = new List<string>();
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (line.Contains(expectedString))
-                {
-                    lines.Add(line);
-                }
-                line = streamReader.ReadLine();
-            }
-
-            return lines.ToArray(); ;
+            return FindLinesThatContain([expectedString]);
         }
 
         public string[] FindLinesThatContain(params string[] tags)
         {
             var result = new List<string>();
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (tags.All(line.Contains))
-                {
-                    result.Add(line);
-                }
-
-                line = streamReader.ReadLine();
-            }
-
+            IterateLines(result.Add, tags);
             return result.ToArray();
         }
 

Framework/KubernetesWorkflow/K8sController.cs

@@ -12,11 +12,10 @@ namespace KubernetesWorkflow
         private readonly ILog log;
         private readonly K8sCluster cluster;
         private readonly WorkflowNumberSource workflowNumberSource;
-        private readonly Func<string?, string?> replacer;
         private readonly K8sClient client;
         public const string PodLabelKey = "pod-uuid";
 
-        public K8sController(ILog log, K8sCluster cluster, WorkflowNumberSource workflowNumberSource, string k8sNamespace, Func<string?, string?> replacer)
+        public K8sController(ILog log, K8sCluster cluster, WorkflowNumberSource workflowNumberSource, string k8sNamespace)
         {
             this.log = log;
             this.cluster = cluster;
@@ -24,7 +23,6 @@ namespace KubernetesWorkflow
             client = new K8sClient(cluster.GetK8sClientConfig());
 
             K8sNamespace = k8sNamespace;
-            this.replacer = replacer;
         }
 
         public void Dispose()
@@ -66,7 +64,7 @@ namespace KubernetesWorkflow
             if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
         }
 
-        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous, Func<string?, string?> replacer)
+        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous)
         {
             log.Debug();
 
@@ -74,7 +72,7 @@ namespace KubernetesWorkflow
             var recipeName = container.Recipe.Name;
 
             using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines, previous: previous));
-            logHandler.Log(stream, replacer);
+            logHandler.Log(stream);
         }
 
         public string ExecuteCommand(RunningContainer container, string command, params string[] args)
@@ -908,7 +906,7 @@ namespace KubernetesWorkflow
                 var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
                 log.Error(msg);
 
-                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true, replacer);
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg, deploymentName), tailLines: null, previous: true);
 
                 throw new Exception(msg);
             }
@@ -954,7 +952,7 @@ namespace KubernetesWorkflow
             var podName = GetPodName(container);
             var recipeName = container.Recipe.Name;
 
-            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace, replacer);
+            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
         }
 
         private V1Pod[] FindPodsByLabel(string podLabel)

Framework/KubernetesWorkflow/LogHandler.cs

@@ -4,19 +4,19 @@ namespace KubernetesWorkflow
 {
     public interface ILogHandler
     {
-        void Log(Stream log, Func<string?, string?> replacer);
+        void Log(Stream log);
     }
 
     public abstract class LogHandler : ILogHandler
     {
-        public void Log(Stream log, Func<string?, string?> replacer)
+        public void Log(Stream log)
         {
             using var reader = new StreamReader(log);
             var line = reader.ReadLine();
             while (line != null)
             {
-                line = replacer(reader.ReadLine());
-                if (line != null) ProcessLine(line);
+                ProcessLine(line);
+                line = reader.ReadLine();
             }
         }
 
@@ -25,9 +25,9 @@ namespace KubernetesWorkflow
 
     public class WriteToFileLogHandler : LogHandler, ILogHandler
     {
-        public WriteToFileLogHandler(ILog sourceLog, string description)
+        public WriteToFileLogHandler(ILog sourceLog, string description, string addFileName)
         {
-            LogFile = sourceLog.CreateSubfile();
+            LogFile = sourceLog.CreateSubfile(addFileName);
 
             var msg = $"{description} -->> {LogFile.FullFilename}";
             sourceLog.Log(msg);

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -28,17 +28,16 @@ namespace KubernetesWorkflow
         private readonly WorkflowNumberSource numberSource;
         private readonly K8sCluster cluster;
         private readonly string k8sNamespace;
-        private readonly Func<string?, string?> replacer;
         private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
         private readonly LocationProvider locationProvider;
 
-        internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, string k8sNamespace, Func<string?, string?> replacer)
+        internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, string k8sNamespace)
         {
             this.log = log;
             this.numberSource = numberSource;
             this.cluster = cluster;
             this.k8sNamespace = k8sNamespace;
-            this.replacer = replacer;
+
             locationProvider = new LocationProvider(log, K8s);
         }
 
@@ -120,7 +119,7 @@ namespace KubernetesWorkflow
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container, logHandler, tailLines, previous, replacer);
+                controller.DownloadPodLog(container, logHandler, tailLines, previous);
             });
         }
 
@@ -128,11 +127,11 @@ namespace KubernetesWorkflow
         {
             var msg = $"Downloading container log for '{container.Name}'";
             log.Log(msg);
-            var logHandler = new WriteToFileLogHandler(log, msg);
+            var logHandler = new WriteToFileLogHandler(log, msg, container.Name);
 
             K8s(controller =>
             {
-                controller.DownloadPodLog(container, logHandler, tailLines, previous, replacer);
+                controller.DownloadPodLog(container, logHandler, tailLines, previous);
             });
 
             return new DownloadedLog(logHandler, container.Name);
@@ -258,7 +257,7 @@ namespace KubernetesWorkflow
         {
             try
             {
-                var controller = new K8sController(log, cluster, numberSource, k8sNamespace, replacer);
+                var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
                 action(controller);
                 controller.Dispose();
             }
@@ -273,7 +272,7 @@ namespace KubernetesWorkflow
         {
             try
             {
-                var controller = new K8sController(log, cluster, numberSource, k8sNamespace, replacer);
+                var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
                 var result = action(controller);
                 controller.Dispose();
                 return result;

Framework/KubernetesWorkflow/WorkflowCreator.cs

@@ -25,7 +25,7 @@ namespace KubernetesWorkflow
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
                                                                     containerNumberSource);
 
-            return new StartupWorkflow(log, workflowNumberSource, cluster, GetNamespace(namespaceOverride), configuration.Replacer);
+            return new StartupWorkflow(log, workflowNumberSource, cluster, GetNamespace(namespaceOverride));
         }
 
         private string GetNamespace(string? namespaceOverride)

Framework/Logging/BaseLog.cs

@@ -8,7 +8,7 @@ namespace Logging
         void Debug(string message = "", int skipFrames = 0);
         void Error(string message);
         void AddStringReplace(string from, string to);
-        LogFile CreateSubfile(string ext = "log");
+        LogFile CreateSubfile(string addName, string ext = "log");
     }
 
     public abstract class BaseLog : ILog
@@ -72,9 +72,13 @@ namespace Logging
             File.Delete(LogFile.FullFilename);
         }
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
+            addName = addName
+                .Replace("<", "")
+                .Replace(">", "");
+
+            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}_{addName}", ext);
         }
 
         protected string ApplyReplacements(string str)

Framework/Logging/LogFile.cs

@@ -1,4 +1,6 @@
-namespace Logging
+using Utils;
+
+namespace Logging
 {
     public class LogFile
     {
@@ -49,7 +51,7 @@
 
         private static string GetTimestamp()
         {
-            return $"[{DateTime.UtcNow.ToString("o")}]";
+            return $"[{Time.FormatTimestamp(DateTime.UtcNow)}]";
         }
 
         private void EnsurePathExists(string filename)

Framework/Logging/LogPrefixer.cs

@@ -18,9 +18,9 @@
         public string Prefix { get; set; } = string.Empty;
 
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return backingLog.CreateSubfile(ext);
+            return backingLog.CreateSubfile(addName, ext);
         }
 
         public void Debug(string message = "", int skipFrames = 0)

Framework/Logging/LogSplitter.cs

@@ -14,9 +14,9 @@
             OnAll(l => l.AddStringReplace(from, to));
         }
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return targetLogs.First().CreateSubfile(ext);
+            return targetLogs.First().CreateSubfile(addName, ext);
         }
 
         public void Debug(string message = "", int skipFrames = 0)

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -1,25 +1,25 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace NethereumWorkflow
 {
     public class NethereumInteraction
     {
-        // BlockCache is a static instance: It stays alive for the duration of the application runtime.
-        private readonly static BlockCache blockCache = new BlockCache();
+        private readonly BlockCache blockCache;
 
         private readonly ILog log;
         private readonly Web3 web3;
 
-        internal NethereumInteraction(ILog log, Web3 web3)
+        internal NethereumInteraction(ILog log, Web3 web3, BlockCache blockCache)
         {
             this.log = log;
             this.web3 = web3;
+            this.blockCache = blockCache;
         }
 
         public string SendEth(string toAddress, decimal ethAmount)

Framework/NethereumWorkflow/NethereumInteractionCreator.cs

@@ -1,4 +1,5 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.Web3;
 
 namespace NethereumWorkflow
@@ -6,13 +7,15 @@ namespace NethereumWorkflow
     public class NethereumInteractionCreator
     {
         private readonly ILog log;
+        private readonly BlockCache blockCache;
         private readonly string ip;
         private readonly int port;
         private readonly string privateKey;
 
-        public NethereumInteractionCreator(ILog log, string ip, int port, string privateKey)
+        public NethereumInteractionCreator(ILog log, BlockCache blockCache, string ip, int port, string privateKey)
         {
             this.log = log;
+            this.blockCache = blockCache;
             this.ip = ip;
             this.port = port;
             this.privateKey = privateKey;
@@ -21,7 +24,7 @@ namespace NethereumWorkflow
         public NethereumInteraction CreateWorkflow()
         {
             log.Debug("Starting interaction to " + ip + ":" + port);
-            return new NethereumInteraction(log, CreateWeb3());
+            return new NethereumInteraction(log, CreateWeb3(), blockCache);
         }
 
         private Web3 CreateWeb3()

Framework/NethereumWorkflow/NethereumWorkflow.csproj

@@ -12,6 +12,7 @@
 	</ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\BlockchainUtils\BlockchainUtils.csproj" />
     <ProjectReference Include="..\Logging\Logging.csproj" />
     <ProjectReference Include="..\Utils\Utils.csproj" />
   </ItemGroup>

Framework/NethereumWorkflow/Web3Wrapper.cs

@@ -1,16 +1,11 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
 using Utils;
 
 namespace NethereumWorkflow
 {
-    public interface IWeb3Blocks
-    {
-        ulong GetCurrentBlockNumber();
-        DateTime? GetTimestampForBlock(ulong blockNumber);
-    }
-
     public class Web3Wrapper : IWeb3Blocks
     {
         private readonly Web3 web3;

Framework/OverwatchTranscript/EventBucketReader.cs

@@ -7,6 +7,7 @@ namespace OverwatchTranscript
     public interface IFinalizedBucket
     {
         bool IsEmpty { get; }
+        void Update();
         DateTime? SeeTopUtc();
         BucketTop? TakeTop();
     }
@@ -28,7 +29,8 @@ namespace OverwatchTranscript
         private readonly string bucketFile;
         private readonly ConcurrentQueue<BucketTop> topQueue = new ConcurrentQueue<BucketTop>();
         private readonly AutoResetEvent itemDequeued = new AutoResetEvent(false);
-        private bool stopping;
+        private readonly AutoResetEvent itemEnqueued = new AutoResetEvent(false);
+        private bool sourceIsEmpty;
 
         public EventBucketReader(ILog log, string bucketFile)
         {
@@ -42,34 +44,38 @@ namespace OverwatchTranscript
         
         public bool IsEmpty { get; private set; }
 
+        public void Update()
+        {
+            if (IsEmpty) return;
+            while (topQueue.Count == 0)
+            {
+                UpdateIsEmpty();
+                if (IsEmpty) return;
+
+                itemDequeued.Set();
+                itemEnqueued.WaitOne(200);
+            }
+        }
+
         public DateTime? SeeTopUtc()
         {
             if (IsEmpty) return null;
-            while (true)
+            if (topQueue.TryPeek(out BucketTop? top))
             {
-                UpdateIsEmpty();
-                if (IsEmpty) return null;
-                if (topQueue.TryPeek(out BucketTop? top))
-                {
-                    return top.Utc;
-                }
+                return top.Utc;
             }
+            return null;
         }
 
         public BucketTop? TakeTop()
         {
             if (IsEmpty) return null;
-
-            while (true)
+            if (topQueue.TryDequeue(out BucketTop? top))
             {
-                UpdateIsEmpty();
-                if (IsEmpty) return null;
-                if (topQueue.TryDequeue(out BucketTop? top))
-                {
-                    itemDequeued.Set();
-                    return top;
-                }
+                itemDequeued.Set();
+                return top;
             }
+            return null;
         }
 
         private void ReadBucket()
@@ -85,23 +91,25 @@ namespace OverwatchTranscript
                     if (top != null)
                     {
                         topQueue.Enqueue(top);
+                        itemEnqueued.Set();
                     }
                     else
                     {
-                        stopping = true;
+                        sourceIsEmpty = true;
+                        UpdateIsEmpty();
                         return;
                     }
                 }
 
                 itemDequeued.Reset();
-                itemDequeued.WaitOne();
+                itemDequeued.WaitOne(5000);
             }
         }
 
         private void UpdateIsEmpty()
         {
-            var empty = stopping && topQueue.IsEmpty;
-            if (!IsEmpty && empty)
+            var allEmpty = sourceIsEmpty && topQueue.IsEmpty;
+            if (!IsEmpty && allEmpty)
             {
                 File.Delete(bucketFile);
                 IsEmpty = true;

Framework/OverwatchTranscript/MomentReferenceBuilder.cs

@@ -24,6 +24,8 @@ namespace OverwatchTranscript
             log.Debug($"Building references for {buckets.Count} buckets.");
             while (buckets.Any())
             {
+                foreach (var b in buckets) b.Update();
+
                 buckets.RemoveAll(b => b.IsEmpty);
                 if (!buckets.Any()) break;
 

Framework/Utils/BlockInterval.cs

@@ -15,12 +15,13 @@
                 To = from;
             }
             TimeRange = timeRange;
+            NumberOfBlocks = (To - From) + 1;
         }
 
         public ulong From { get; }
         public ulong To { get; }
         public TimeRange TimeRange { get; }
-        public ulong NumberOfBlocks => To - From;
+        public ulong NumberOfBlocks { get; }
 
         public override string ToString()
         {

Framework/Utils/PluginPathUtils.cs

@@ -0,0 +1,41 @@
+namespace Utils
+{
+    public static class PluginPathUtils
+    {
+        private const string ProjectPluginsFolderName = "ProjectPlugins";
+        private static string projectPluginsDir = string.Empty;
+
+        public static string ProjectPluginsDir
+        {
+            get
+            {
+                if (string.IsNullOrEmpty(projectPluginsDir)) projectPluginsDir = FindProjectPluginsDir();
+                return projectPluginsDir;
+            }
+        }
+
+        private static string FindProjectPluginsDir()
+        {
+            var current = Directory.GetCurrentDirectory();
+            while (true)
+            {
+                var localFolders = Directory.GetDirectories(current);
+                var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
+                if (projectPluginsFolders.Length == 1)
+                {
+                    return projectPluginsFolders.Single();
+                }
+
+                var parent = Directory.GetParent(current);
+                if (parent == null)
+                {
+                    var msg = $"Unable to locate '{ProjectPluginsFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
+                    Console.WriteLine(msg);
+                    throw new Exception(msg);
+                }
+
+                current = parent.FullName;
+            }
+        }
+    }
+}

Framework/Utils/RandomUtils.cs

@@ -5,6 +5,16 @@
         private static readonly Random random = new Random();
         private static readonly object @lock = new object();
 
+        public static T GetOneRandom<T>(this T[] items)
+        {
+            lock (@lock)
+            {
+                var i = random.Next(0, items.Length);
+                var result = items[i];
+                return result;
+            }
+        }
+
         public static T PickOneRandom<T>(this List<T> remainingItems)
         {
             lock (@lock)

Framework/Utils/Retry.cs

@@ -71,6 +71,10 @@
                         task();
                         return;
                     }
+                    catch (OperationCanceledException)
+                    {
+                        return;
+                    }
                     catch (Exception ex)
                     {
                         var failure = CaptureFailure(ex);
@@ -94,7 +98,7 @@
 
             private void Fail()
             {
-                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport}",
+                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport()}",
                         new AggregateException(failures.Select(f => f.Exception)));
             }
 

Framework/Utils/Time.cs

@@ -33,6 +33,11 @@
             result += $"{d.Seconds} secs";
             return result;
         }
+
+        public static string FormatTimestamp(DateTime d)
+        {
+            return d.ToString("o");
+        }
         
         public static TimeSpan ParseTimespan(string span)
         {

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin.Marketplace;
+using System.Collections.Generic;
 using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
@@ -12,7 +13,8 @@ namespace CodexContractsPlugin.ChainMonitor
             RequestCancelledEventDTO[] cancelled,
             RequestFailedEventDTO[] failed,
             SlotFilledEventDTO[] slotFilled,
-            SlotFreedEventDTO[] slotFreed
+            SlotFreedEventDTO[] slotFreed,
+            SlotReservationsFullEventDTO[] slotReservationsFull
             )
         {
             BlockInterval = blockInterval;
@@ -22,6 +24,9 @@ namespace CodexContractsPlugin.ChainMonitor
             Failed = failed;
             SlotFilled = slotFilled;
             SlotFreed = slotFreed;
+            SlotReservationsFull = slotReservationsFull;
+
+            All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull);
         }
 
         public BlockInterval BlockInterval { get; }
@@ -31,21 +36,8 @@ namespace CodexContractsPlugin.ChainMonitor
         public RequestFailedEventDTO[] Failed { get; }
         public SlotFilledEventDTO[] SlotFilled { get; }
         public SlotFreedEventDTO[] SlotFreed { get; }
-
-        public IHasBlock[] All
-        {
-            get
-            {
-                var all = new List<IHasBlock>();
-                all.AddRange(Requests);
-                all.AddRange(Fulfilled);
-                all.AddRange(Cancelled);
-                all.AddRange(Failed);
-                all.AddRange(SlotFilled);
-                all.AddRange(SlotFreed);
-                return all.ToArray();
-            }
-        }
+        public SlotReservationsFullEventDTO[] SlotReservationsFull { get; }
+        public IHasBlock[] All { get; }
 
         public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
         {
@@ -66,8 +58,19 @@ namespace CodexContractsPlugin.ChainMonitor
                 events.GetRequestCancelledEvents(),
                 events.GetRequestFailedEvents(),
                 events.GetSlotFilledEvents(),
-                events.GetSlotFreedEvents()
+                events.GetSlotFreedEvents(),
+                events.GetSlotReservationsFull()
             );
         }
+
+        private T[] ConcatAll<T>(params T[][] arrays)
+        {
+            var result = Array.Empty<T>();
+            foreach (var array in arrays)
+            {
+                result = result.Concat(array).ToArray();
+            }
+            return result;
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -1,7 +1,7 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
-using NethereumWorkflow.BlockUtils;
 using System.Numerics;
 using Utils;
 
@@ -16,6 +16,9 @@ namespace CodexContractsPlugin.ChainMonitor
         void OnRequestFailed(RequestEvent requestEvent);
         void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
         void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
+        void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
+
+        void OnError(string msg);
     }
 
     public class RequestEvent
@@ -66,13 +69,18 @@ namespace CodexContractsPlugin.ChainMonitor
         private void Apply(ChainEvents events)
         {
             if (events.BlockInterval.TimeRange.From < TotalSpan.From)
-                throw new Exception("Attempt to update ChainState with set of events from before its current record.");
+            {
+                var msg = "Attempt to update ChainState with set of events from before its current record.";
+                handler.OnError(msg);
+                throw new Exception(msg);
+            }
 
-            log.Log($"ChainState updating: {events.BlockInterval}");
+            log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
 
             // Run through each block and apply the events to the state in order.
             var span = events.BlockInterval.TimeRange.Duration;
             var numBlocks = events.BlockInterval.NumberOfBlocks;
+            if (numBlocks == 0) return;
             var spanPerBlock = span / numBlocks;
 
             var eventUtc = events.BlockInterval.TimeRange.From;
@@ -109,7 +117,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFulfilledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
             handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
@@ -117,7 +125,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestCancelledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
             handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
@@ -125,7 +133,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFailedEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Failed);
             handler.OnRequestFailed(new RequestEvent(@event.Block, r));
@@ -133,7 +141,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFilledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
@@ -142,13 +150,21 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFreedEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.Hosts.RemoveHost((int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
             handler.OnSlotFreed(new RequestEvent(@event.Block, r), @event.SlotIndex);
         }
 
+        private void ApplyEvent(SlotReservationsFullEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.Log($"[{@event.Block.BlockNumber}] SlotReservationsFull (slotIndex:{@event.SlotIndex})");
+            handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
+        }
+
         private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
         {
             foreach (var r in requests)
@@ -162,10 +178,23 @@ namespace CodexContractsPlugin.ChainMonitor
             }
         }
 
-        private ChainStateRequest? FindRequest(byte[] requestId)
+        private ChainStateRequest? FindRequest(IHasRequestId request)
         {
-            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
-            if (r == null) log.Log("Unable to find request by ID!");
+            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
+            if (r == null)
+            {
+                var blockNumber = "unknown";
+                if (request is IHasBlock blk)
+                {
+                    blockNumber = blk.Block.BlockNumber.ToString();
+                }
+
+                var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
+                    $"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
+
+                log.Error(msg);
+                handler.OnError(msg);
+            }
             return r;
         }
 

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -1,10 +1,5 @@
 using GethPlugin;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Numerics;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
@@ -51,5 +46,15 @@ namespace CodexContractsPlugin.ChainMonitor
         {
             foreach (var handler in Handlers) handler.OnSlotFreed(requestEvent, slotIndex);
         }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotReservationsFull(requestEvent, slotIndex);
+        }
+
+        public void OnError(string msg)
+        {
+            foreach (var handler in Handlers) handler.OnError(msg);
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -32,5 +32,13 @@ namespace CodexContractsPlugin.ChainMonitor
         public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
         }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public void OnError(string msg)
+        {
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -1,7 +1,9 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.ABI;
+using Nethereum.Hex.HexConvertors.Extensions;
 using Nethereum.Util;
 using NethereumWorkflow;
 using Newtonsoft.Json;
@@ -24,6 +26,7 @@ namespace CodexContractsPlugin
         ICodexContractsEvents GetEvents(BlockInterval blockInterval);
         EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
         RequestState GetRequestState(Request request);
+        void WaitUntilNextPeriod();
     }
 
     [JsonConverter(typeof(StringEnumConverter))]
@@ -114,6 +117,15 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
+        public void WaitUntilNextPeriod()
+        {
+            log.Log("Waiting until next proof period...");
+            var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
+            var periodSeconds = (int)Deployment.Config.Proofs.Period;
+            var secondsLeft = now % periodSeconds;
+            Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1));
+        }
+
         private ContractInteractions StartInteraction()
         {
             return new ContractInteractions(log, gethNode);

ProjectPlugins/CodexContractsPlugin/CodexContractsDeployment.cs

@@ -1,14 +1,18 @@
-namespace CodexContractsPlugin
+using CodexContractsPlugin.Marketplace;
+
+namespace CodexContractsPlugin
 {
     public class CodexContractsDeployment
     {
-        public CodexContractsDeployment(string marketplaceAddress, string abi, string tokenAddress)
+        public CodexContractsDeployment(MarketplaceConfig config, string marketplaceAddress, string abi, string tokenAddress)
         {
+            Config = config;
             MarketplaceAddress = marketplaceAddress;
             Abi = abi;
             TokenAddress = tokenAddress;
         }
 
+        public MarketplaceConfig Config { get; }
         public string MarketplaceAddress { get; }
         public string Abi { get; }
         public string TokenAddress { get; }

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -1,8 +1,9 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
+using Nethereum.Contracts;
 using Nethereum.Hex.HexTypes;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -16,6 +17,7 @@ namespace CodexContractsPlugin
         RequestFailedEventDTO[] GetRequestFailedEvents();
         SlotFilledEventDTO[] GetSlotFilledEvents();
         SlotFreedEventDTO[] GetSlotFreedEvents();
+        SlotReservationsFullEventDTO[] GetSlotReservationsFull();
     }
 
     public class CodexContractsEvents : ICodexContractsEvents
@@ -38,49 +40,32 @@ namespace CodexContractsPlugin
         {
             var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
             var i = new ContractInteractions(log, gethNode);
-            return events
-            .Select(e =>
+            return events.Select(e =>
             {
-                        var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
-                        var result = requestEvent.ReturnValue1;
-                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                        result.RequestId = e.Event.RequestId;
-                        return result;
-                    })
-                    .ToArray();
+                var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
+                var result = requestEvent.ReturnValue1;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.RequestId = e.Event.RequestId;
+                return result;
+            }).ToArray();
         }
 
         public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()
         {
             var events = gethNode.GetEvents<RequestFulfilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public RequestCancelledEventDTO[] GetRequestCancelledEvents()
         {
             var events = gethNode.GetEvents<RequestCancelledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public RequestFailedEventDTO[] GetRequestFailedEvents()
         {
             var events = gethNode.GetEvents<RequestFailedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public SlotFilledEventDTO[] GetSlotFilledEvents()
@@ -98,12 +83,20 @@ namespace CodexContractsPlugin
         public SlotFreedEventDTO[] GetSlotFreedEvents()
         {
             var events = gethNode.GetEvents<SlotFreedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        public SlotReservationsFullEventDTO[] GetSlotReservationsFull()
+        {
+            var events = gethNode.GetEvents<SlotReservationsFullEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
+        {
+            var result = e.Event;
+            result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+            return result;
         }
 
         private BlockTimeEntry GetBlock(ulong number)

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -4,6 +4,7 @@ using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
+using Newtonsoft.Json;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -34,6 +35,7 @@ namespace CodexContractsPlugin
             try
             {
                 var result = DeployContract(container, workflow, gethNode);
+
                 workflow.Stop(containers, waitTillStopped: false);
                 Log("Container stopped.");
                 return result;
@@ -75,9 +77,20 @@ namespace CodexContractsPlugin
 
             Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract));
 
-            Log("Synced. Codex SmartContracts deployed.");
+            Log("Synced. Codex SmartContracts deployed. Getting configuration...");
 
-            return new CodexContractsDeployment(marketplaceAddress, abi, tokenAddress);
+            var config = GetMarketplaceConfiguration(marketplaceAddress, gethNode);
+
+            Log("Got config: " + JsonConvert.SerializeObject(config));
+
+            return new CodexContractsDeployment(config, marketplaceAddress, abi, tokenAddress);
+        }
+
+        private MarketplaceConfig GetMarketplaceConfiguration(string marketplaceAddress, IGethNode gethNode)
+        {
+            var func = new ConfigurationFunctionBase();
+            var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
+            return response.ReturnValue1;
         }
 
         private void EnsureCompatbility(string abi, string bytecode)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
@@ -23,9 +24,8 @@ namespace CodexContractsPlugin
         public string GetTokenAddress(string marketplaceAddress)
         {
             log.Debug(marketplaceAddress);
-            var function = new GetTokenFunction();
-
-            return gethNode.Call<GetTokenFunction, string>(marketplaceAddress, function);
+            var function = new TokenFunctionBase();
+            return gethNode.Call<TokenFunctionBase, string>(marketplaceAddress, function);
         }
 
         public string GetTokenName(string tokenAddress)
@@ -111,11 +111,6 @@ namespace CodexContractsPlugin
         }
     }
 
-    [Function("token", "address")]
-    public class GetTokenFunction : FunctionMessage
-    {
-    }
-
     [Function("name", "string")]
     public class GetTokenNameFunction : FunctionMessage
     {

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,6 +1,6 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
+using BlockchainUtils;
 using GethPlugin;
-using NethereumWorkflow.BlockUtils;
 using Newtonsoft.Json;
 
 namespace CodexContractsPlugin.Marketplace
@@ -10,7 +10,12 @@ namespace CodexContractsPlugin.Marketplace
         BlockTimeEntry Block { get; set; }
     }
 
-    public partial class Request : RequestBase, IHasBlock
+    public interface IHasRequestId
+    {
+        byte[] RequestId { get; set; }
+    }
+
+    public partial class Request : RequestBase, IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
@@ -28,32 +33,38 @@ namespace CodexContractsPlugin.Marketplace
         }
     }
 
-    public partial class RequestFulfilledEventDTO : IHasBlock
+    public partial class RequestFulfilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestCancelledEventDTO : IHasBlock
+    public partial class RequestCancelledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestFailedEventDTO : IHasBlock
+    public partial class RequestFailedEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class SlotFilledEventDTO : IHasBlock
+    public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
     }
 
-    public partial class SlotFreedEventDTO : IHasBlock
+    public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
+
+    public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }

ProjectPlugins/CodexContractsPlugin/SelfUpdater.cs

@@ -1,4 +1,6 @@
-namespace CodexContractsPlugin
+using Utils;
+
+namespace CodexContractsPlugin
 {
     public class SelfUpdater
     {
@@ -41,24 +43,10 @@
 
         private string GetMarketplaceFilePath()
         {
-            var here = Directory.GetCurrentDirectory();
-            while (true)
-            {
-                var path = GetMarketplaceFile(here);
-                if (path != null) return path;
-
-                var parent = Directory.GetParent(here);
-                var up = parent?.FullName;
-                if (up == null || up == here) throw new Exception("Unable to locate ProjectPlugins folder. Unable to update contracts.");
-                here = up;
-            }
-        }
-
-        private string? GetMarketplaceFile(string root)
-        {
-            var path = Path.Combine(root, "ProjectPlugins", "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
-            if (File.Exists(path)) return path;
-            return null;
+            var projectPluginDir = PluginPathUtils.ProjectPluginsDir;
+            var path = Path.Combine(projectPluginDir, "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
+            if (!File.Exists(path)) throw new Exception("Marketplace file not found. Expected: " + path);
+            return path;
         }
 
         private string GenerateContent(string abi, string bytecode)

ProjectPlugins/CodexContractsPlugin/TestTokenExtensions.cs

@@ -46,6 +46,16 @@ namespace CodexContractsPlugin
             return new TestToken(a.TstWei + b.TstWei);
         }
 
+        public static TestToken operator -(TestToken a, TestToken b)
+        {
+            return new TestToken(a.TstWei - b.TstWei);
+        }
+
+        public static TestToken operator *(TestToken a, int b)
+        {
+            return new TestToken(a.TstWei * b);
+        }
+
         public static bool operator <(TestToken a, TestToken b)
         {
             return a.TstWei < b.TstWei;

ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin
     public class RewarderBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-rewarder";
-        public override string Image => "codexstorage/codex-rewarderbot:sha-fb25372";
+        public override string Image => "codexstorage/codex-rewarderbot:sha-8033da1";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -3,13 +3,14 @@ using KubernetesWorkflow.Types;
 using Logging;
 using System.Security.Cryptography;
 using System.Text;
+using Utils;
 
 namespace CodexPlugin
 {
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "6B-94-24-A4-D5-01-6F-12-E9-34-74-36-80-57-7A-3A-79-8C-E8-02-68-B7-05-DA-50-A0-5C-B1-02-B9-AE-C6";
+        private const string OpenApiYamlHash = "34-B5-DA-26-40-76-B8-D8-8E-7D-9C-17-85-C6-B0-63-55-8D-C6-01-0B-96-BB-7C-BD-53-E5-32-07-ED-29-92";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 
@@ -21,8 +22,9 @@ namespace CodexPlugin
 
         private const string Failure =
             "Codex API compatibility check failed! " +
-            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. Please update the openapi.yaml in " +
-            "'ProjectPlugins/CodexPlugin' and rebuild this project. If you wish to disable API compatibility checking, please set " +
+            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. The openapi.yaml in " +
+            "'ProjectPlugins/CodexPlugin' has been overwritten with the container one. " +
+            "Please and rebuild this project. If you wish to disable API compatibility checking, please set " +
             $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/CodexPlugin/ApiChecker.cs'.";
 
         private static bool checkPassed = false;
@@ -71,10 +73,23 @@ namespace CodexPlugin
                 return;
             }
 
+            OverwriteOpenApiYaml(containerApi);
+
             log.Error(Failure);
             throw new Exception(Failure);
         }
 
+        private void OverwriteOpenApiYaml(string containerApi)
+        {
+            Log("API compatibility check failed. Updating CodexPlugin...");
+            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin", "openapi.yaml");
+            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexPlugin/openapi.yaml. Expected: " + openApiFilePath);
+
+            File.Delete(openApiFilePath);
+            File.WriteAllText(openApiFilePath, containerApi);
+            Log("CodexPlugin/openapi.yaml has been updated.");
+        }
+
         private string Hash(string file)
         {
             var fileBytes = Encoding.ASCII.GetBytes(file

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -32,6 +32,22 @@ namespace CodexPlugin
             return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
         }
 
+        public string GetSpr()
+        {
+            return CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                var json = endpoint.HttpGetString("spr");
+                var response = JsonConvert.DeserializeObject<SprResponse>(json);
+                return response!.Spr;
+            });
+        }
+
+        private class SprResponse
+        {
+            public string Spr { get; set; } = string.Empty;
+        }
+
         public DebugPeer GetDebugPeer(string peerId)
         {
             // Cannot use openAPI: debug/peer endpoint is not specified there.
@@ -63,50 +79,75 @@ namespace CodexPlugin
             });
         }
 
-        public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
+        public string UploadFile(UploadInput uploadInput, Action<Failure> onFailure)
         {
             return OnCodex(
-                api => api.UploadAsync(fileStream),
+                api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream),
                 CreateRetryConfig(nameof(UploadFile), onFailure));
         }
 
         public Stream DownloadFile(string contentId, Action<Failure> onFailure)
         {
             var fileResponse = OnCodex(
-                api => api.DownloadNetworkAsync(contentId),
+                api => api.DownloadNetworkStreamAsync(contentId),
                 CreateRetryConfig(nameof(DownloadFile), onFailure));
 
             if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
             return fileResponse.Stream;
         }
 
+        public LocalDataset DownloadStreamless(ContentId cid)
+        {
+            var response = OnCodex(api => api.DownloadNetworkAsync(cid.Id));
+            return mapper.Map(response);
+        }
+
+        public LocalDataset DownloadManifestOnly(ContentId cid)
+        {
+            var response = OnCodex(api => api.DownloadNetworkManifestAsync(cid.Id));
+            return mapper.Map(response);
+        }
+
         public LocalDatasetList LocalFiles()
         {
-            return mapper.Map(OnCodex(api => api.ListDataAsync()));
+            // API for listData mismatches.
+            //return mapper.Map(OnCodex(api => api.ListDataAsync()));
+
+            return mapper.Map(CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                return Time.Retry(() =>
+                {
+                    var str = endpoint.HttpGetString("data");
+                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
+                    return JsonConvert.DeserializeObject<LocalDatasetListJson>(str)!;
+                }, nameof(LocalFiles));
+            }));
+
         }
 
         public StorageAvailability SalesAvailability(StorageAvailability request)
         {
             var body = mapper.Map(request);
-            var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
+            var read = OnCodex(api => api.OfferStorageAsync(body));
             return mapper.Map(read);
         }
 
         public StorageAvailability[] GetAvailabilities()
         {
-            var collection = OnCodex<ICollection<SalesAvailabilityREAD>>(api => api.GetAvailabilitiesAsync());
+            var collection = OnCodex(api => api.GetAvailabilitiesAsync());
             return mapper.Map(collection);
         }
 
         public string RequestStorage(StoragePurchaseRequest request)
         {
             var body = mapper.Map(request);
-            return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
+            return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
         public CodexSpace Space()
         {
-            var space = OnCodex<Space>(api => api.SpaceAsync());
+            var space = OnCodex(api => api.SpaceAsync());
             return mapper.Map(space);
         }
 
@@ -193,8 +234,15 @@ namespace CodexPlugin
                 .CreateEndpoint(GetAddress(), "/api/codex/v1/", Container.Name);
         }
 
+        public static Address? UploaderOverride { get; set; } = null;
+        public static Address? DownloaderOverride { get; set; } = null;
+
         private Address GetAddress()
         {
+            if (GetName().ToLowerInvariant().Contains("upload") && UploaderOverride != null) return UploaderOverride;
+            if (GetName().ToLowerInvariant().Contains("download") && DownloaderOverride != null) return DownloaderOverride;
+
+
             return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
         }
 
@@ -261,4 +309,18 @@ namespace CodexPlugin
             log.Log($"{GetName()} {msg}");
         }
     }
+
+    public class UploadInput
+    {
+        public UploadInput(string contentType, string contentDisposition, FileStream fileStream)
+        {
+            ContentType = contentType;
+            ContentDisposition = contentDisposition;
+            FileStream = fileStream;
+        }
+
+        public string ContentType { get; }
+        public string ContentDisposition { get; }
+        public FileStream FileStream { get; }
+    }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,8 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "thatbenbierens/nim-codex:netpeerdebug6";
-            //"codexstorage/nim-codex:0.1.4";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";
@@ -110,7 +109,7 @@ namespace CodexPlugin
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
                 var account = marketplaceSetup.EthAccountSetup.GetNew();
-                AddEnvVar("PRIV_KEY", account.PrivateKey);
+                AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
                 Additional(account);
 
                 SetCommandOverride(marketplaceSetup);

ProjectPlugins/CodexPlugin/CodexLogLine.cs

@@ -48,6 +48,11 @@ namespace CodexPlugin
         public string Message { get; set; } = string.Empty;
         public Dictionary<string, string> Attributes { get; private set; } = new Dictionary<string, string>();
 
+        public override string ToString()
+        {
+            return Message;
+        }
+
         /// <summary>
         /// After too much time spent cursing at regexes, here's what I got:
         /// Parses input string into 'key=value' pair, considerate of quoted (") values.

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -14,12 +14,23 @@ namespace CodexPlugin
     {
         string GetName();
         string GetPeerId();
-        DebugInfo GetDebugInfo();
+        DebugInfo GetDebugInfo(bool log = false);
+        string GetSpr();
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
         ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
+        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
+        (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, string fileLabel = "");
+        (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
+        LocalDataset DownloadStreamless(ContentId cid);
+        /// <summary>
+        /// TODO: This will monitor the quota-used of the node until 'size' bytes are added. That's a very bad way
+        /// to track the streamless download progress. Replace it once we have a good API for this.
+        /// </summary>
+        LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size);
+        LocalDataset DownloadManifestOnly(ContentId cid);
         LocalDatasetList LocalFiles();
         CodexSpace Space();
         void ConnectToPeer(ICodexNode node);
@@ -119,14 +130,22 @@ namespace CodexPlugin
             return peerId;
         }
 
-        public DebugInfo GetDebugInfo()
+        public DebugInfo GetDebugInfo(bool log = false)
         {
             var debugInfo = CodexAccess.GetDebugInfo();
-            var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
-            Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
+            if (log)
+            {
+                var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
+                Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
+            }
             return debugInfo;
         }
 
+        public string GetSpr()
+        {
+            return CodexAccess.GetSpr();
+        }
+
         public DebugPeer GetDebugPeer(string peerId)
         {
             return CodexAccess.GetDebugPeer(peerId);
@@ -138,6 +157,11 @@ namespace CodexPlugin
         }
 
         public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
+        {
+            return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"", onFailure);
+        }
+
+        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenRead(file.Filename);
             var uniqueId = Guid.NewGuid().ToString();
@@ -145,10 +169,11 @@ namespace CodexPlugin
 
             hooks.OnFileUploading(uniqueId, size);
 
-            var logMessage = $"Uploading file {file.Describe()}...";
+            var input = new UploadInput(contentType, contentDisposition, fileStream);
+            var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
             var measurement = Stopwatch.Measure(log, logMessage, () =>
             {
-                return CodexAccess.UploadFile(fileStream, onFailure);
+                return CodexAccess.UploadFile(input, onFailure);
             });
 
             var response = measurement.Value;
@@ -166,10 +191,20 @@ namespace CodexPlugin
 
         public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
         {
-            return DownloadContent(contentId, DoNothing, fileLabel);
+            return DownloadContentT(contentId, fileLabel).Item1;
         }
 
         public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
+        {
+            return DownloadContentT(contentId, onFailure, fileLabel).Item1;
+        }
+
+        public (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, string fileLabel = "")
+        {
+            return DownloadContentT(contentId, DoNothing, fileLabel);
+        }
+
+        public (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
         {
             var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
             hooks.OnFileDownloading(contentId);
@@ -182,7 +217,34 @@ namespace CodexPlugin
             transferSpeeds.AddDownloadSample(size, measurement);
             hooks.OnFileDownloaded(size, contentId);
 
-            return file;
+            return (file, measurement);
+        }
+
+        public LocalDataset DownloadStreamless(ContentId cid)
+        {
+            Log($"Downloading streamless '{cid}' (no-wait)");
+            return CodexAccess.DownloadStreamless(cid);
+        }
+
+        public LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size)
+        {
+            Log($"Downloading streamless '{cid}' (wait till finished)");
+
+            var sw = Stopwatch.Measure(log, nameof(DownloadStreamlessWait), () =>
+            {
+                var startSpace = Space();
+                var result = CodexAccess.DownloadStreamless(cid);
+                WaitUntilQuotaUsedIncreased(startSpace, size);
+                return result;
+            });
+
+            return sw.Value;
+        }
+
+        public LocalDataset DownloadManifestOnly(ContentId cid)
+        {
+            Log($"Downloading manifest-only '{cid}'");
+            return CodexAccess.DownloadManifestOnly(cid);
         }
 
         public LocalDatasetList LocalFiles()
@@ -264,10 +326,27 @@ namespace CodexPlugin
         private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenWrite(file.Filename);
+            var timeout = tools.TimeSet.HttpCallTimeout();
             try
             {
-                using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
-                downloadStream.CopyTo(fileStream);
+                // Type of stream generated by openAPI client does not support timeouts.
+                var start = DateTime.UtcNow;
+                var cts = new CancellationTokenSource();
+                var downloadTask = Task.Run(() =>
+                {
+                    using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
+                    downloadStream.CopyTo(fileStream);
+                }, cts.Token);
+                
+                while (DateTime.UtcNow - start < timeout)
+                {
+                    if (downloadTask.IsFaulted) throw downloadTask.Exception;
+                    if (downloadTask.IsCompletedSuccessfully) return;
+                    Thread.Sleep(100);
+                }
+
+                cts.Cancel();
+                throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
             }
             catch
             {
@@ -276,6 +355,39 @@ namespace CodexPlugin
             }
         }
 
+        public void WaitUntilQuotaUsedIncreased(CodexSpace startSpace, ByteSize expectedIncreaseOfQuotaUsed)
+        {
+            WaitUntilQuotaUsedIncreased(startSpace, expectedIncreaseOfQuotaUsed, TimeSpan.FromMinutes(2));
+        }
+
+        public void WaitUntilQuotaUsedIncreased(
+            CodexSpace startSpace,
+            ByteSize expectedIncreaseOfQuotaUsed,
+            TimeSpan maxTimeout)
+        {
+            Log($"Waiting until quotaUsed " +
+                $"(start: {startSpace.QuotaUsedBytes}) " +
+                $"increases by {expectedIncreaseOfQuotaUsed} " +
+                $"to reach {startSpace.QuotaUsedBytes + expectedIncreaseOfQuotaUsed.SizeInBytes}");
+
+            var retry = new Retry($"Checking local space for quotaUsed increase of {expectedIncreaseOfQuotaUsed}",
+            maxTimeout: maxTimeout,
+            sleepAfterFail: TimeSpan.FromSeconds(3),
+            onFail: f => { });
+
+            retry.Run(() =>
+            {
+                var space = Space();
+                var increase = space.QuotaUsedBytes - startSpace.QuotaUsedBytes;
+
+                if (increase < expectedIncreaseOfQuotaUsed.SizeInBytes)
+                    throw new Exception($"Expected quota-used not reached. " +
+                        $"Expected increase: {expectedIncreaseOfQuotaUsed.SizeInBytes} " +
+                        $"Actual increase: {increase} " +
+                        $"Actual used: {space.QuotaUsedBytes}");
+            });
+        }
+
         private void EnsureMarketplace()
         {
             if (ethAccount == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -20,7 +20,7 @@ namespace CodexPlugin
 
         public void Announce()
         {
-            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
+            Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
         }
 
         public void AddMetadata(IAddMetadata metadata)
@@ -55,6 +55,10 @@ namespace CodexPlugin
             {
                 mconfig.GethNode.SendEth(node, mconfig.MarketplaceSetup.InitialEth);
                 mconfig.CodexContracts.MintTestTokens(node, mconfig.MarketplaceSetup.InitialTestTokens);
+
+                Log($"Send {mconfig.MarketplaceSetup.InitialEth} and " +
+                    $"minted {mconfig.MarketplaceSetup.InitialTestTokens} for " +
+                    $"{node.GetName()} (address: {node.EthAddress})");
             }
         }
 
@@ -70,5 +74,10 @@ namespace CodexPlugin
             setup(codexSetup);
             return codexSetup;
         }
+
+        private void Log(string msg)
+        {
+            tools.GetLog().Log(msg);
+        }
     }
 }

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -16,8 +16,16 @@ namespace CodexPlugin
                 Spr = debugInfo.Spr,
                 Addrs = debugInfo.Addrs.ToArray(),
                 AnnounceAddresses = JArray(debugInfo.AdditionalProperties, "announceAddresses").Select(x => x.ToString()).ToArray(),
-                Version = MapDebugInfoVersion(JObject(debugInfo.AdditionalProperties, "codex")),
-                Table = MapDebugInfoTable(JObject(debugInfo.AdditionalProperties, "table"))
+                Version = Map(debugInfo.Codex),
+                Table = Map(debugInfo.Table)
+            };
+        }
+
+        public LocalDatasetList Map(LocalDatasetListJson json)
+        {
+            return new LocalDatasetList
+            {
+                Content = json.Content.Select(Map).ToArray()
             };
         }
 
@@ -38,6 +46,15 @@ namespace CodexPlugin
             };
         }
 
+        public LocalDataset Map(LocalDatasetListJsonItem item)
+        {
+            return new LocalDataset
+            {
+                Cid = new ContentId(item.Cid),
+                Manifest = MapManifest(item.Manifest)
+            };
+        }
+
         public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
         {
             return new CodexOpenApi.SalesAvailabilityCREATE
@@ -136,47 +153,45 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoVersion MapDebugInfoVersion(JObject obj)
+        private DebugInfoVersion Map(CodexVersion obj)
         {
             return new DebugInfoVersion
             {
-                Version = StringOrEmpty(obj, "version"),
-                Revision = StringOrEmpty(obj, "revision")
+                Version = obj.Version,
+                Revision = obj.Revision
             };
         }
 
-        private DebugInfoTable MapDebugInfoTable(JObject obj)
+        private DebugInfoTable Map(PeersTable obj)
         {
             return new DebugInfoTable
             {
-                LocalNode = MapDebugInfoTableNode(obj.GetValue("localNode")),
-                Nodes = MapDebugInfoTableNodeArray(obj.GetValue("nodes") as JArray)
+                LocalNode = Map(obj.LocalNode),
+                Nodes = Map(obj.Nodes)
             };
         }
 
-        private DebugInfoTableNode MapDebugInfoTableNode(JToken? token)
+        private DebugInfoTableNode Map(Node? token)
         {
-            var obj = token as JObject;
-            if (obj == null) return new DebugInfoTableNode();
-
+            if (token == null) return new DebugInfoTableNode();
             return new DebugInfoTableNode
             {
-                Address = StringOrEmpty(obj, "address"),
-                NodeId = StringOrEmpty(obj, "nodeId"),
-                PeerId = StringOrEmpty(obj, "peerId"),
-                Record = StringOrEmpty(obj, "record"),
-                Seen = Bool(obj, "seen")
+                Address = token.Address,
+                NodeId = token.NodeId,
+                PeerId = token.PeerId,
+                Record = token.Record,
+                Seen = token.Seen
             };
         }
 
-        private DebugInfoTableNode[] MapDebugInfoTableNodeArray(JArray? nodes)
+        private DebugInfoTableNode[] Map(ICollection<Node> nodes)
         {
             if (nodes == null || nodes.Count == 0)
             {
                 return new DebugInfoTableNode[0];
             }
 
-            return nodes.Select(MapDebugInfoTableNode).ToArray();
+            return nodes.Select(Map).ToArray();
         }
 
         private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)
@@ -184,8 +199,20 @@ namespace CodexPlugin
             return new Manifest
             {
                 BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
-                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.OriginalBytes)),
-                RootHash = manifest.RootHash,
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
+                RootHash = manifest.TreeCid,
+                Protected = manifest.Protected
+            };
+        }
+
+        public Manifest MapManifest(LocalDatasetListJsonItemManifest manifest)
+        {
+            return new Manifest
+            {
+                // needs update
+                BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
+                RootHash = manifest.TreeCid,
                 Protected = manifest.Protected
             };
         }
@@ -245,4 +272,42 @@ namespace CodexPlugin
             return new ByteSize(Convert.ToInt64(size));
         }
     }
+
+
+//"content": [
+//        {
+//            "cid": "zDvZRwzkxLxVaGces3kpkHjo8EcTPXudvYMfNxdoH21Ask1Js5fJ",
+//            "manifest": {
+//                "treeCid": "zDzSvJTf8GBRyEDNuAzXS9VnRfh8cNuYuRPwTLW6RUQReSgKnhCt",
+//                "datasetSize": 5242880,
+//                "blockSize": 65536,
+//                "filename": null,
+//                "mimetype": "application/octet-stream",
+//                "uploadedAt": 1731426230,
+//                "protected": false
+//            }
+//        }
+//    ]
+
+    public class LocalDatasetListJson
+    {
+        public LocalDatasetListJsonItem[] Content { get; set; } = Array.Empty<LocalDatasetListJsonItem>();
+    }
+
+    public class LocalDatasetListJsonItem
+    {
+        public string Cid { get; set; } = string.Empty;
+        public LocalDatasetListJsonItemManifest Manifest { get; set; } = new();
+    }
+
+    public class LocalDatasetListJsonItemManifest
+    {
+        public string TreeCid { get; set; } = string.Empty;
+        public int DatasetSize { get; set; }
+        public int BlockSize { get; set; }
+        public string? Filename { get; set; } = string.Empty;
+        public string? MimeType { get; set; } = string.Empty;
+        public int? UploadedAt { get; set; } 
+        public bool Protected { get; set; }
+    }
 }

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -1,7 +1,5 @@
 using CodexContractsPlugin;
-using CodexOpenApi;
 using Logging;
-using System.Data;
 using Utils;
 
 namespace CodexPlugin
@@ -40,6 +38,12 @@ namespace CodexPlugin
         public string State { get; set; } = string.Empty;
         public string Error { get; set; } = string.Empty;
         public StorageRequest Request { get; set; } = null!;
+
+        public bool IsCancelled => State.ToLowerInvariant().Contains("cancel");
+        public bool IsError => State.ToLowerInvariant().Contains("error");
+        public bool IsFinished => State.ToLowerInvariant().Contains("finished");
+        public bool IsStarted => State.ToLowerInvariant().Contains("started");
+        public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
     }
 
     public class StorageRequest

ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs

@@ -1,4 +1,6 @@
-using CodexPlugin.Hooks;
+using CodexContractsPlugin;
+using CodexPlugin.Hooks;
+using GethPlugin;
 using Logging;
 using Newtonsoft.Json;
 using Utils;
@@ -12,7 +14,8 @@ namespace CodexPlugin
         ContentId ContentId { get; }
         void WaitForStorageContractSubmitted();
         void WaitForStorageContractStarted();
-        void WaitForStorageContractFinished();
+        void WaitForStorageContractFinished(ICodexContracts contracts);
+        void WaitForContractFailed();
     }
 
     public class StoragePurchaseContract : IStoragePurchaseContract
@@ -62,7 +65,7 @@ namespace CodexPlugin
             AssertDuration(SubmittedToStarted, timeout, nameof(SubmittedToStarted));
         }
 
-        public void WaitForStorageContractFinished()
+        public void WaitForStorageContractFinished(ICodexContracts contracts)
         {
             if (!contractStartedUtc.HasValue)
             {
@@ -74,6 +77,24 @@ namespace CodexPlugin
             contractFinishedUtc = DateTime.UtcNow;
             LogFinishedDuration();
             AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
+
+            contracts.WaitUntilNextPeriod();
+            contracts.WaitUntilNextPeriod();
+
+            var blocks = 3;
+            Log($"Waiting {blocks} blocks for nodes to process payouts...");
+            Thread.Sleep(GethContainerRecipe.BlockInterval * blocks);
+        }
+
+        public void WaitForContractFailed()
+        {
+            if (!contractStartedUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
+            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
+            WaitForStorageContractState(timeout, "failed");
         }
 
         public StoragePurchase GetPurchaseStatus(string purchaseId)

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -83,33 +83,46 @@ components:
         id:
           $ref: "#/components/schemas/PeerId"
 
-    ErasureParameters:
-      type: object
-      properties:
-        totalChunks:
-          type: integer
-
-    PoRParameters:
-      description: Parameters for Proof of Retrievability
-      type: object
-      properties:
-        u:
-          type: string
-        publicKey:
-          type: string
-        name:
-          type: string
-
     Content:
       type: object
       description: Parameters specifying the content
       properties:
         cid:
           $ref: "#/components/schemas/Cid"
-        erasure:
-          $ref: "#/components/schemas/ErasureParameters"
-        por:
-          $ref: "#/components/schemas/PoRParameters"
+
+    Node:
+      type: object
+      properties:
+        nodeId:
+          type: string
+        peerId:
+          type: string
+        record:
+          type: string
+        address:
+          type: string
+        seen:
+          type: boolean
+
+    CodexVersion:
+      type: object
+      properties:
+        version:
+          type: string
+          example: v0.1.7
+        revision:
+          type: string
+          example: 0c647d8
+
+    PeersTable:
+      type: object
+      properties:
+        localNode:
+          $ref: "#/components/schemas/Node"
+        nodes:
+          type: array
+          items:
+            $ref: "#/components/schemas/Node"
 
     DebugInfo:
       type: object
@@ -125,6 +138,10 @@ components:
           description: Path of the data repository where all nodes data are stored
         spr:
           $ref: "#/components/schemas/SPR"
+        table:
+          $ref: "#/components/schemas/PeersTable"
+        codex:
+          $ref: "#/components/schemas/CodexVersion"
 
     SalesAvailability:
       type: object
@@ -327,10 +344,10 @@ components:
     ManifestItem:
       type: object
       properties:
-        rootHash:
+        treeCid:
           $ref: "#/components/schemas/Cid"
-          description: "Root hash of the content"
-        originalBytes:
+          description: "Unique data identifier"
+        datasetSize:
           type: integer
           format: int64
           description: "Length of original content in bytes"
@@ -340,6 +357,22 @@ components:
         protected:
           type: boolean
           description: "Indicates if content is protected by erasure-coding"
+        filename:
+          type: string
+          nullable: true
+          description: "The original name of the uploaded content (optional)"
+          example: codex.png
+        mimetype:
+          type: string
+          nullable: true
+          description: "The original mimetype of the uploaded content (optional)"
+          example: image/png
+        uploadedAt:
+          type: integer
+          format: int64
+          nullable: true
+          description: "The UTC upload timestamp in seconds"
+          example: 1729244192
 
     Space:
       type: object
@@ -351,15 +384,15 @@ components:
         quotaMaxBytes:
           type: integer
           format: int64
-          description: "Maximum storage space used by the node"
+          description: "Maximum storage space (in bytes) available for the node in Codex's local repository."
         quotaUsedBytes:
           type: integer
           format: int64
-          description: "Amount of storage space currently in use"
+          description: "Amount of storage space (in bytes) currently used for storing files in Codex's local repository."
         quotaReservedBytes:
           type: integer
           format: int64
-          description: "Amount of storage space reserved"
+          description: "Amount of storage reserved (in bytes) in the Codex's local repository for future use when storage requests will be picked up and hosted by the node using node's availabilities. This does not include the storage currently in use."
 
 servers:
   - url: "http://localhost:8080/api/codex/v1"
@@ -425,12 +458,29 @@ paths:
           description: Invalid CID is specified
         "404":
           description: Content specified by the CID is not found
+        "422":
+          description: The content type is not a valid content type or the filename is not valid
         "500":
           description: Well it was bad-bad
     post:
       summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
       tags: [ Data ]
       operationId: upload
+      parameters:
+        - name: content-type
+          in: header
+          required: false
+          description: The content type of the file. Must be valid.
+          schema:
+            type: string
+            example: "image/png"
+        - name: content-disposition
+          in: header
+          required: false
+          description: The content disposition used to send the filename.
+          schema:
+            type: string
+            example: "attachment; filename=\"codex.png\""
       requestBody:
         content:
           application/octet-stream:
@@ -476,10 +526,36 @@ paths:
           description: Well it was bad-bad
 
   "/data/{cid}/network":
+    post:
+      summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
+      tags: [ Data ]
+      operationId: downloadNetwork
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File to be downloaded."
+      responses:
+        "200":
+          description: Manifest information for download that has been started.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
+  "/data/{cid}/network/stream":
     get:
       summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
       tags: [ Data ]
-      operationId: downloadNetwork
+      operationId: downloadNetworkStream
       parameters:
         - in: path
           name: cid
@@ -502,6 +578,32 @@ paths:
         "500":
           description: Well it was bad-bad
 
+  "/data/{cid}/network/manifest":
+    get:
+      summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
+      tags: [ Data ]
+      operationId: downloadNetworkManifest
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File for which the manifest is to be downloaded."
+      responses:
+        "200":
+          description: Manifest information.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
   "/space":
     get:
       summary: "Gets a summary of the storage space allocation of the node."
@@ -747,7 +849,7 @@ paths:
         "503":
           description: Persistence is not enabled
 
-  "/node/spr":
+  "/spr":
     get:
       summary: "Get Node's SPR"
       operationId: getSPR
@@ -765,7 +867,7 @@ paths:
         "503":
           description: Node SPR not ready, try again later
 
-  "/node/peerid":
+  "/peerid":
     get:
       summary: "Get Node's PeerID"
       operationId: getPeerId
@@ -813,4 +915,4 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/DebugInfo"
+                $ref: "#/components/schemas/DebugInfo"

ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj

@@ -7,4 +7,8 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+  </ItemGroup>
+
 </Project>

ProjectPlugins/CodexPluginPrebuild/Program.cs

@@ -1,5 +1,6 @@
 using System.Security.Cryptography;
 using System.Text;
+using Utils;
 
 public static class Program
 {
@@ -40,32 +41,9 @@ public static class Program
 
     private static string FindCodexPluginFolder()
     {
-        var current = Directory.GetCurrentDirectory();
-
-        while (true)
-        {
-            var localFolders = Directory.GetDirectories(current);
-            var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
-            if (projectPluginsFolders.Length == 1)
-            {
-                return Path.Combine(projectPluginsFolders.Single(), CodexPluginFolderName);
-            }
-            var codexPluginFolders = localFolders.Where(l => l.EndsWith(CodexPluginFolderName)).ToArray();
-            if (codexPluginFolders.Length == 1)
-            {
-                return codexPluginFolders.Single();
-            }
-
-            var parent = Directory.GetParent(current);
-            if (parent == null)
-            {
-                var msg = $"Unable to locate '{CodexPluginFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
-                Console.WriteLine(msg);
-                throw new Exception(msg);
-            }
-
-            current = parent.FullName;
-        }
+        var folder = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin");
+        if (!Directory.Exists(folder)) throw new Exception("CodexPlugin folder not found. Expected: " + folder);
+        return folder;
     }
 
     private static string CreateHash(string openApiFile)

ProjectPlugins/GethPlugin/CoreInterfaceExtensions.cs

@@ -1,4 +1,5 @@
-using Core;
+using BlockchainUtils;
+using Core;
 
 namespace GethPlugin
 {
@@ -9,15 +10,15 @@ namespace GethPlugin
             return Plugin(ci).DeployGeth(setup);
         }
 
-        public static IGethNode WrapGethDeployment(this CoreInterface ci, GethDeployment deployment)
+        public static IGethNode WrapGethDeployment(this CoreInterface ci, GethDeployment deployment, BlockCache blockCache)
         {
-            return Plugin(ci).WrapGethDeployment(deployment);
+            return Plugin(ci).WrapGethDeployment(deployment, blockCache);
         }
 
-        public static IGethNode StartGethNode(this CoreInterface ci, Action<IGethSetup> setup)
+        public static IGethNode StartGethNode(this CoreInterface ci, BlockCache blockCache, Action<IGethSetup> setup)
         {
             var deploy = DeployGeth(ci, setup);
-            return WrapGethDeployment(ci, deploy);
+            return WrapGethDeployment(ci, deploy, blockCache);
         }
 
         private static GethPlugin Plugin(CoreInterface ci)

ProjectPlugins/GethPlugin/GethContainerRecipe.cs

@@ -6,6 +6,7 @@ namespace GethPlugin
     public class GethContainerRecipe : ContainerRecipeFactory
     {
         public static string DockerImage { get; } = "codexstorage/dist-tests-geth:latest";
+        public static TimeSpan BlockInterval { get; } = TimeSpan.FromSeconds(1.0);
         private const string defaultArgs = "--ipcdisable --syncmode full";
 
         public const string HttpPortTag = "http_port";

ProjectPlugins/GethPlugin/GethNode.cs

@@ -1,11 +1,11 @@
-using Core;
+using BlockchainUtils;
+using Core;
 using KubernetesWorkflow.Types;
 using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using NethereumWorkflow;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace GethPlugin
@@ -34,10 +34,12 @@ namespace GethPlugin
     public class DeploymentGethNode : BaseGethNode, IGethNode
     {
         private readonly ILog log;
+        private readonly BlockCache blockCache;
 
-        public DeploymentGethNode(ILog log, GethDeployment startResult)
+        public DeploymentGethNode(ILog log, BlockCache blockCache, GethDeployment startResult)
         {
             this.log = log;
+            this.blockCache = blockCache;
             StartResult = startResult;
         }
 
@@ -60,7 +62,7 @@ namespace GethPlugin
             var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
             var account = StartResult.Account;
 
-            var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);
+            var creator = new NethereumInteractionCreator(log, blockCache, address.Host, address.Port, account.PrivateKey);
             return creator.CreateWorkflow();
         }
     }
@@ -68,6 +70,7 @@ namespace GethPlugin
     public class CustomGethNode : BaseGethNode, IGethNode
     {
         private readonly ILog log;
+        private readonly BlockCache blockCache;
         private readonly string gethHost;
         private readonly int gethPort;
         private readonly string privateKey;
@@ -75,9 +78,10 @@ namespace GethPlugin
         public GethDeployment StartResult => throw new NotImplementedException();
         public RunningContainer Container => throw new NotImplementedException();
 
-        public CustomGethNode(ILog log, string gethHost, int gethPort, string privateKey)
+        public CustomGethNode(ILog log, BlockCache blockCache, string gethHost, int gethPort, string privateKey)
         {
             this.log = log;
+            this.blockCache = blockCache;
             this.gethHost = gethHost;
             this.gethPort = gethPort;
             this.privateKey = privateKey;
@@ -90,7 +94,7 @@ namespace GethPlugin
 
         protected override NethereumInteraction StartInteraction()
         {
-            var creator = new NethereumInteractionCreator(log, gethHost, gethPort, privateKey);
+            var creator = new NethereumInteractionCreator(log, blockCache, gethHost, gethPort, privateKey);
             return creator.CreateWorkflow();
         }
     }

ProjectPlugins/GethPlugin/GethPlugin.cs

@@ -1,4 +1,5 @@
-using Core;
+using BlockchainUtils;
+using Core;
 
 namespace GethPlugin
 {
@@ -36,10 +37,10 @@ namespace GethPlugin
             return starter.StartGeth(startupConfig);
         }
 
-        public IGethNode WrapGethDeployment(GethDeployment startResult)
+        public IGethNode WrapGethDeployment(GethDeployment startResult, BlockCache blockCache)
         {
             startResult = SerializeGate.Gate(startResult);
-            return starter.WrapGethContainer(startResult);
+            return starter.WrapGethContainer(startResult, blockCache);
         }
     }
 }

ProjectPlugins/GethPlugin/GethStarter.cs

@@ -1,4 +1,5 @@
-using Core;
+using BlockchainUtils;
+using Core;
 using KubernetesWorkflow;
 
 namespace GethPlugin
@@ -41,10 +42,10 @@ namespace GethPlugin
             return new GethDeployment(containers, discoveryPort, httpPort, wsPort, account, pubKey);
         }
 
-        public IGethNode WrapGethContainer(GethDeployment startResult)
+        public IGethNode WrapGethContainer(GethDeployment startResult, BlockCache blockCache)
         {
             startResult = SerializeGate.Gate(startResult);
-            return new DeploymentGethNode(tools.GetLog(), startResult);
+            return new DeploymentGethNode(tools.GetLog(), blockCache, startResult);
         }
 
         private void Log(string msg)

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -6,14 +6,14 @@ namespace MetricsPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray(), scrapeInterval);
         }
 
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets);
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets, scrapeInterval);
         }
 
         public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
@@ -26,19 +26,19 @@ namespace MetricsPlugin
             return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
         {
-            return ci.GetMetricsFor(manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return ci.GetMetricsFor(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            var rc = ci.DeployMetricsCollector(scrapeTargets);
+            var rc = ci.DeployMetricsCollector(scrapeInterval, scrapeTargets);
             return scrapeTargets.Select(t => ci.WrapMetricsCollector(rc, t)).ToArray();
         }
 

ProjectPlugins/MetricsPlugin/MetricsAccess.cs

@@ -7,7 +7,7 @@ namespace MetricsPlugin
     public interface IMetricsAccess : IHasContainer
     {
         string TargetName { get; }
-        Metrics? GetAllMetrics();
+        Metrics GetAllMetrics();
         MetricsSet GetMetric(string metricName);
         MetricsSet GetMetric(string metricName, TimeSpan timeout);
     }
@@ -27,7 +27,7 @@ namespace MetricsPlugin
         public string TargetName { get; }
         public RunningContainer Container => query.RunningContainer;
 
-        public Metrics? GetAllMetrics()
+        public Metrics GetAllMetrics()
         {
             return query.GetAllMetricsForNode(target);
         }
@@ -54,11 +54,10 @@ namespace MetricsPlugin
             }
         }
 
-        private MetricsSet? GetMostRecent(string metricName)
+        private MetricsSet GetMostRecent(string metricName)
         {
             var result = query.GetMostRecent(metricName, target);
-            if (result == null) return null;
-            return result.Sets.LastOrDefault();
+            return result.Sets.Last();
         }
     }
 }

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -31,9 +31,9 @@ namespace MetricsPlugin
         {
         }
 
-        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
+        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets, TimeSpan scrapeInterval)
         {
-            return starter.CollectMetricsFor(scrapeTargets);
+            return starter.CollectMetricsFor(scrapeTargets, scrapeInterval);
         }
 
         public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -23,10 +23,10 @@ namespace MetricsPlugin
 
         public RunningContainer RunningContainer { get; }
 
-        public Metrics? GetMostRecent(string metricName, IMetricsScrapeTarget target)
+        public Metrics GetMostRecent(string metricName, IMetricsScrapeTarget target)
         {
             var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
-            if (response == null) return null;
+            if (response == null) throw new Exception($"Failed to get most recent metric: {metricName}");
 
             var result = new Metrics
             {
@@ -44,19 +44,20 @@ namespace MetricsPlugin
             return result;
         }
 
-        public Metrics? GetMetrics(string metricName)
+        public Metrics GetMetrics(string metricName)
         {
             var response = GetAll(metricName);
-            if (response == null) return null;
+            if (response == null) throw new Exception($"Failed to get metrics by name: {metricName}");
             var result = MapResponseToMetrics(response);
             Log(metricName, result);
             return result;
         }
 
-        public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
+        public Metrics GetAllMetricsForNode(IMetricsScrapeTarget target)
         {
-            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
-            if (response.status != "success") return null;
+            var instanceString = GetInstanceStringForNode(target);
+            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={instanceString}{GetQueryTimeRange()}");
+            if (response.status != "success") throw new Exception($"Failed to get metrics for target: {instanceString}");
             var result = MapResponseToMetrics(response);
             Log(target, result);
             return result;
@@ -80,18 +81,32 @@ namespace MetricsPlugin
         {
             return new Metrics
             {
-                Sets = response.data.result.Select(r =>
-                {
-                    return new MetricsSet
-                    {
-                        Name = r.metric.__name__,
-                        Instance = r.metric.instance,
-                        Values = MapMultipleValues(r.values)
-                    };
-                }).ToArray()
+                Sets = response.data.result.Select(CreateMetricsSet).ToArray()
             };
         }
 
+        private MetricsSet CreateMetricsSet(PrometheusQueryResponseDataResultEntry r)
+        {
+            var result = new MetricsSet
+            {
+                Name = r.metric.__name__,
+                Instance = r.metric.instance,
+                Values = MapMultipleValues(r.values)
+            };
+
+            if (!string.IsNullOrEmpty(r.metric.file) && !string.IsNullOrEmpty(r.metric.line) && !string.IsNullOrEmpty(r.metric.proc))
+            {
+                result.AsyncProfiler = new AsyncProfilerMetrics
+                {
+                    File = r.metric.file,
+                    Line = r.metric.line,
+                    Proc = r.metric.proc
+                };
+            }
+
+            return result;
+        }
+
         private MetricsSetValue[] MapSingleValue(object[] value)
         {
             if (value != null && value.Length > 0)
@@ -220,14 +235,28 @@ namespace MetricsPlugin
     {
         public string Name { get; set; } = string.Empty;
         public string Instance { get; set; } = string.Empty;
+        public AsyncProfilerMetrics? AsyncProfiler { get; set; } = null;
         public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
 
         public override string ToString()
         {
-            return $"{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
+            var prefix = "";
+            if (AsyncProfiler != null)
+            {
+                prefix = $"proc: '{AsyncProfiler.Proc}' in '{AsyncProfiler.File}:{AsyncProfiler.Line}'";
+            }
+
+            return $"{prefix}{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
         }
     }
 
+    public class AsyncProfilerMetrics
+    {
+        public string File { get; set; } = string.Empty;
+        public string Line { get; set; } = string.Empty;
+        public string Proc { get; set; } = string.Empty;
+    }
+
     public class MetricsSetValue
     {
         public DateTime Timestamp { get; set; }
@@ -263,6 +292,10 @@ namespace MetricsPlugin
         public string __name__ { get; set; } = string.Empty;
         public string instance { get; set; } = string.Empty;
         public string job { get; set; } = string.Empty;
+        // Async profiler output.
+        public string? file { get; set; } = null;
+        public string? line { get; set; } = null;
+        public string? proc { get; set; } = null;
     }
 
     public class PrometheusAllNamesResponse

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -16,13 +16,13 @@ namespace MetricsPlugin
             this.tools = tools;
         }
 
-        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets)
+        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
         {
             if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
 
             Log($"Starting metrics server for {targets.Length} targets...");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets, scrapeInterval)));
 
             var workflow = tools.CreateWorkflow();
             var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
@@ -48,12 +48,16 @@ namespace MetricsPlugin
             tools.GetLog().Log(msg);
         }
 
-        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets)
+        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
         {
+            var secs = Convert.ToInt32(scrapeInterval.TotalSeconds);
+            if (secs < 1) throw new Exception("ScrapeInterval can't be < 1s");
+            if (secs > 60) throw new Exception("ScrapeInterval can't be > 60s");
+
             var config = "";
             config += "global:\n";
-            config += "  scrape_interval: 10s\n";
-            config += "  scrape_timeout: 10s\n";
+            config += $"  scrape_interval: {secs}s\n";
+            config += $"  scrape_timeout: {secs}s\n";
             config += "\n";
             config += "scrape_configs:\n";
             config += "  - job_name: services\n";

README.md

@@ -2,7 +2,7 @@
 
 This project allows you to write tools and tests that control and interact with container-based applications to form a distributed system in a controlled, reproducible environment.
 
-Dotnet: v7.0  
+Dotnet: v8.0  
 Kubernetes: v1.25.4  
 Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp  
 Nethereum: v4.14.0

SpeedCheckTests/SpeedCheckTests.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+	<ItemGroup>
+		<PackageReference Include="nunit" Version="3.13.3" />
+		<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
+		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+	</ItemGroup>
+
+	<ItemGroup>
+		<ProjectReference Include="..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
+		<ProjectReference Include="..\Tests\DistTestCore\DistTestCore.csproj" />
+		<ProjectReference Include="..\Tests\ExperimentalTests\ExperimentalTests.csproj" />
+	</ItemGroup>
+</Project>

SpeedCheckTests/SpeedTest.cs

@@ -0,0 +1,107 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System.Diagnostics;
+using System.Drawing;
+using Utils;
+
+namespace SpeedCheckTests
+{
+    [TestFixture]
+    public class SpeedTest : CodexDistTest
+    {
+        [Test]
+        public void Symmetric()
+        {
+            // Symmetric: A node always sends a reply to every message it receives.
+            CodexContainerRecipe.DockerImageOverride = "thatbenbierens/nim-codex:blkex-cancelpresence-27-f";
+
+            var uploader = StartCodex(s => s.WithName("SymUploader"));
+            var downloader = StartCodex(s => s.WithName("SymDownloader").WithBootstrapNode(uploader));
+            var timeTaken = PerformTest(uploader, downloader);
+
+            Console.WriteLine($"Symmetric time: {Time.FormatDuration(timeTaken)}");
+
+            Assert.That(timeTaken, Is.LessThan(TimeSpan.FromSeconds(10.0)), 
+                $"Symmetric: Too slow. Expected less than 10 seconds but was: {Time.FormatDuration(timeTaken)}");
+        }
+
+        [Test]
+        public void Asymmetric()
+        {
+            // Asymmetric: A node does not always send a reply when a message is received.
+            CodexContainerRecipe.DockerImageOverride = "thatbenbierens/nim-codex:blkex-cancelpresence-27-s";
+
+            var uploader = StartCodex(s => s.WithName("AsymUploader"));
+            var downloader = StartCodex(s => s.WithName("AsymDownloader").WithBootstrapNode(uploader));
+            var timeTaken = PerformTest(uploader, downloader);
+
+            Console.WriteLine($"Asymmetric time: {Time.FormatDuration(timeTaken)}");
+
+            Assert.That(timeTaken, Is.LessThan(TimeSpan.FromSeconds(10.0)),
+                $"Asymmetric: Too slow. Expected less than 10 seconds but was: {Time.FormatDuration(timeTaken)}");
+        }
+
+        [Test]
+        public void Binary()
+        {
+            // Docker image not used: Here for api check.
+            CodexContainerRecipe.DockerImageOverride = "thatbenbierens/nim-codex:blkex-cancelpresence-27-f";
+
+            var binary = "C:\\Projects\\nim-codex\\build\\codex.exe";
+            if (!File.Exists(binary)) throw new Exception("TODO: Update binary path");
+
+            var uploadInfo = new ProcessStartInfo
+            {
+                FileName = binary,
+                Arguments = "--data-dir=upload_data " +
+                "--api-port=8081 " +
+                "--nat=127.0.0.1 " +
+                "--disc-ip=127.0.0.1 " +
+                "--disc-port=8091 " +
+                "--listen-addrs=/ip4/127.0.0.1/tcp/8071",
+                UseShellExecute = true,
+            };
+
+            var uploadProcess = Process.Start(uploadInfo);
+
+            Thread.Sleep(5000);
+            if (uploadProcess == null || uploadProcess.HasExited) throw new Exception("Node exited.");
+
+            CodexAccess.UploaderOverride = new Address("http://localhost", 8081);
+            var uploader = StartCodex(s => s.WithName("BinaryUploader"));
+            var spr = uploader.GetSpr();
+            
+            var downloadProcess = Process.Start(binary,
+                "--data-dir=download_data " +
+                "--api-port=8082 " +
+                "--nat=127.0.0.1 " +
+                "--disc-ip=127.0.0.1 " +
+                "--disc-port=8092 " +
+                "--listen-addrs=/ip4/127.0.0.1/tcp/8072 " +
+                "--bootstrap-node=" + spr
+            );
+
+            CodexAccess.DownloaderOverride = new Address("http://localhost", 8082);
+            var downloader = StartCodex(s => s.WithName("BinaryDownloader"));
+
+            var timeTaken = PerformTest(uploader, downloader);
+
+            uploadProcess.Kill();
+            downloadProcess.Kill();
+
+            Console.WriteLine($"Binary time: {Time.FormatDuration(timeTaken)}");
+
+            Assert.That(timeTaken, Is.LessThan(TimeSpan.FromSeconds(10.0)),
+                $"Binary: Too slow. Expected less than 10 seconds but was: {Time.FormatDuration(timeTaken)}");
+        }
+
+        private TimeSpan PerformTest(ICodexNode uploader, ICodexNode downloader)
+        {
+            var testFile = GenerateTestFile(100.MB());
+            var contentId = uploader.UploadFile(testFile);
+            var (downloadedFile, timeTaken) = downloader.DownloadContentT(contentId);
+            return timeTaken;
+        }
+    }
+}

Tests/CodexContinuousTests/CodexContinuousTests.csproj

@@ -14,8 +14,8 @@
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
-    <ProjectReference Include="..\CodexTests\CodexTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\ExperimentalTests\ExperimentalTests.csproj" />
   </ItemGroup>
 
 </Project>

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -130,7 +130,7 @@ namespace ContinuousTests
                 var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
                 var openingLine =
                     $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}";
-                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart,
+                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(node.GetName()), node.Container, effectiveStart,
                     effectiveEnd, openingLine);
             }
         }

Tests/CodexLongTests/CodexTestsLong.csproj

@@ -14,8 +14,8 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
-    <ProjectReference Include="..\CodexTests\CodexTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\ExperimentalTests\ExperimentalTests.csproj" />
   </ItemGroup>
 
 </Project>

Tests/CodexReleaseTests/CodexReleaseTests.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="NUnit" Version="3.13.3" />
+	<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
+	<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ExperimentalTests\ExperimentalTests.csproj" />
+  </ItemGroup>
+
+</Project>

Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs

@@ -0,0 +1,47 @@
+using CodexPlugin;
+using CodexTests;
+using FileUtils;
+using NUnit.Framework;
+using System.Diagnostics;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    public class InterruptUploadTest : CodexDistTest
+    {
+        [Test]
+        public void UploadInterruptTest()
+        {
+            var nodes = StartCodex(10);
+
+            var tasks = nodes.Select(n => Task<bool>.Run(() => RunInterruptUploadTest(n)));
+            Task.WaitAll(tasks.ToArray());
+
+            Assert.That(tasks.Select(t => t.Result).All(r => r == true));
+
+            WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), nodes);
+        }
+
+        private bool RunInterruptUploadTest(ICodexNode node)
+        {
+            var file = GenerateTestFile(300.MB());
+
+            var process = StartCurlUploadProcess(node, file);
+
+            Thread.Sleep(500);
+            process.Kill();
+            Thread.Sleep(1000);
+
+            var log = Ci.DownloadLog(node);
+            return !log.GetLinesContaining("Unhandled exception in async proc, aborting").Any();
+        }
+
+        private Process StartCurlUploadProcess(ICodexNode node, TrackedFile file)
+        {
+            var apiAddress = node.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
+            var codexUrl = $"{apiAddress}/api/codex/v1/data";
+            var filePath = file.Filename;
+            return Process.Start("curl", $"-X POST {codexUrl} -H \"Content-Type: application/octet-stream\" -T {filePath}");
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/ManifestOnlyDownloadTest.cs

@@ -0,0 +1,33 @@
+using CodexTests;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class ManifestOnlyDownloadTest : CodexDistTest
+    {
+        [Test]
+        public void ManifestOnlyTest()
+        {
+            var uploader = StartCodex();
+            var downloader = StartCodex(s => s.WithBootstrapNode(uploader));
+
+            var file = GenerateTestFile(2.GB());
+            var size = file.GetFilesize().SizeInBytes;
+            var cid = uploader.UploadFile(file);
+
+            var startSpace = downloader.Space();
+            var localDataset = downloader.DownloadManifestOnly(cid);
+
+            Thread.Sleep(1000);
+
+            var spaceDiff = startSpace.FreeBytes - downloader.Space().FreeBytes;
+
+            Assert.That(spaceDiff, Is.LessThan(64.KB().SizeInBytes));
+
+            Assert.That(localDataset.Cid, Is.EqualTo(cid));
+            Assert.That(localDataset.Manifest.OriginalBytes.SizeInBytes, Is.EqualTo(file.GetFilesize().SizeInBytes));
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/OneClientTest.cs

@@ -1,20 +1,26 @@
 using CodexPlugin;
+using CodexTests;
 using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
 using Utils;
 
-namespace CodexTests.BasicTests
+namespace CodexReleaseTests.DataTests
 {
     [TestFixture]
-    public class OneClientTests : CodexDistTest
+    public class OneClientTest : CodexDistTest
     {
         [Test]
-        public void OneClientTest()
+        public void OneClient()
         {
-            var primary = StartCodex();
+            var node = StartCodex();
 
-            PerformOneClientTest(primary);
+            PerformOneClientTest(node);
 
-            LogNodeStatus(primary);
+            LogNodeStatus(node);
         }
 
         private void PerformOneClientTest(ICodexNode primary)
@@ -23,6 +29,8 @@ namespace CodexTests.BasicTests
 
             var contentId = primary.UploadFile(testFile);
 
+            AssertNodesContainFile(contentId, primary);
+
             var downloadedFile = primary.DownloadContent(contentId);
 
             testFile.AssertIsEqual(downloadedFile);

Tests/CodexReleaseTests/DataTests/StreamlessDownloadTest.cs

@@ -0,0 +1,35 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System.Drawing;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class StreamlessDownloadTest : CodexDistTest
+    {
+        [Test]
+        public void StreamlessTest()
+        {
+            var uploader = StartCodex();
+            var downloader = StartCodex(s => s.WithBootstrapNode(uploader));
+
+            var size = 10.MB();
+            var file = GenerateTestFile(size);
+            var cid = uploader.UploadFile(file);
+
+            var startSpace = downloader.Space();
+            var start = DateTime.UtcNow;
+            var localDataset = downloader.DownloadStreamlessWait(cid, size);
+
+            Assert.That(localDataset.Cid, Is.EqualTo(cid));
+            Assert.That(localDataset.Manifest.OriginalBytes.SizeInBytes, Is.EqualTo(file.GetFilesize().SizeInBytes));
+
+            // Stop the uploader node and verify that the downloader has the data.
+            uploader.Stop(waitTillStopped: true);
+            var downloaded = downloader.DownloadContent(cid);
+            file.AssertIsEqual(downloaded);
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/SwarmTest.cs

@@ -0,0 +1,177 @@
+using CodexPlugin;
+using CodexTests;
+using FileUtils;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class SwarmTests : AutoBootstrapDistTest
+    {
+        [Test]
+        [Combinatorial]
+        public void SmallSwarm(
+            [Values(2)] int numberOfNodes,
+            [Values(10)] int filesizeMb
+        )
+        {
+            var filesize = filesizeMb.MB();
+            var nodes = StartCodex(numberOfNodes);
+            var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray();
+
+            var tasks = ParallelDownloadEachFile(nodes, files);
+            Task.WaitAll(tasks);
+
+            AssertAllFilesDownloadedCorrectly(files);
+        }
+
+        [Test]
+        [Combinatorial]
+        public void StreamlessSmallSwarm(
+            [Values(2)] int numberOfNodes,
+            [Values(10)] int filesizeMb
+        )
+        {
+            var filesize = filesizeMb.MB();
+            var nodes = StartCodex(numberOfNodes);
+            var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray();
+
+            var tasks = ParallelStreamlessDownloadEachFile(nodes, files);
+            Task.WaitAll(tasks);
+
+            AssertAllFilesStreamlesslyDownloadedCorrectly(nodes, files);
+        }
+
+        private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node, ByteSize fileSize)
+        {
+            var file = GenerateTestFile(fileSize);
+            var cid = node.UploadFile(file);
+            return new SwarmTestNetworkFile(node, fileSize, file, cid);
+        }
+
+        private Task[] ParallelDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files)
+        {
+            var tasks = new List<Task>();
+
+            foreach (var node in nodes)
+            {
+                tasks.Add(StartDownload(node, files));
+            }
+
+            return tasks.ToArray();
+        }
+
+        private Task[] ParallelStreamlessDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files)
+        {
+            var tasks = new List<Task>();
+
+            foreach (var node in nodes)
+            {
+                tasks.Add(StartStreamlessDownload(node, files));
+            }
+
+            return tasks.ToArray();
+        }
+
+        private Task StartDownload(ICodexNode node, SwarmTestNetworkFile[] files)
+        {
+            return Task.Run(() =>
+            {
+                var remaining = files.ToList();
+
+                while (remaining.Count > 0)
+                {
+                    var file = remaining.PickOneRandom();
+                    try
+                    {
+                        var dl = node.DownloadContent(file.Cid);
+                        lock (file.Lock)
+                        {
+                            file.Downloaded.Add(dl);
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        file.Error = ex;
+                    }
+                }
+            });
+        }
+
+        private Task StartStreamlessDownload(ICodexNode node, SwarmTestNetworkFile[] files)
+        {
+            return Task.Run(() =>
+            {
+                var remaining = files.ToList();
+
+                while (remaining.Count > 0)
+                {
+                    var file = remaining.PickOneRandom();
+                    if (file.Uploader.GetName() != node.GetName())
+                    {
+                        try
+                        {
+                            var startSpace = node.Space();
+                            node.DownloadStreamlessWait(file.Cid, file.OriginalSize);
+                        }
+                        catch (Exception ex)
+                        {
+                            file.Error = ex;
+                        }
+                    }
+                }
+            });
+        }
+
+        private void AssertAllFilesDownloadedCorrectly(SwarmTestNetworkFile[] files)
+        {
+            foreach (var file in files)
+            {
+                if (file.Error != null) throw file.Error;
+                lock (file.Lock)
+                {
+                    foreach (var dl in file.Downloaded)
+                    {
+                        file.Original.AssertIsEqual(dl);
+                    }
+                }
+            }
+        }
+
+        private void AssertAllFilesStreamlesslyDownloadedCorrectly(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files)
+        {
+            var totalFilesSpace = 0.Bytes();
+            foreach (var file in files)
+            {
+                if (file.Error != null) throw file.Error;
+                totalFilesSpace = new ByteSize(totalFilesSpace.SizeInBytes + file.Original.GetFilesize().SizeInBytes);
+            }
+            
+            foreach (var node in nodes)
+            {
+                var currentSpace = node.Space();
+                Assert.That(currentSpace.QuotaUsedBytes, Is.GreaterThanOrEqualTo(totalFilesSpace.SizeInBytes));
+            }
+        }
+
+        private class SwarmTestNetworkFile
+        {
+            public SwarmTestNetworkFile(ICodexNode uploader, ByteSize originalSize, TrackedFile original, ContentId cid)
+            {
+                Uploader = uploader;
+                OriginalSize = originalSize;
+                Original = original;
+                Cid = cid;
+            }
+
+            public ICodexNode Uploader { get; }
+            public ByteSize OriginalSize { get; }
+            public TrackedFile Original { get; }
+            public ContentId Cid { get; }
+            public object Lock { get; } = new object();
+            public List<TrackedFile?> Downloaded { get; } = new List<TrackedFile?>();
+            public Exception? Error { get; set; } = null;
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/ThreeClientTest.cs

@@ -0,0 +1,32 @@
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    public class ThreeClientTest : AutoBootstrapDistTest
+    {
+        [Test]
+        public void ThreeClient()
+        {
+            var primary = StartCodex();
+            var secondary = StartCodex();
+
+            var testFile = GenerateTestFile(10.MB());
+
+            var contentId = primary.UploadFile(testFile);
+            AssertNodesContainFile(contentId, primary);
+
+            var downloadedFile = secondary.DownloadContent(contentId);
+            AssertNodesContainFile(contentId, primary, secondary);
+
+            testFile.AssertIsEqual(downloadedFile);
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/TwoClientTest.cs

@@ -0,0 +1,207 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class TwoClientTests : CodexDistTest
+    {
+        [Test]
+        [Combinatorial]
+        public void TwoClientTest(
+            [Values(
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-2", // S don't send cancel-presence messages
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-1", // F ignore cancel-presence messages
+            //"codexstorage/nim-codex:sha-4b5c355-dist-tests", // F unmodified
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-3", // F same as 1 but logging
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-4", // S no cancel-presence-msg, no fromCancel field
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-5", // F all-presence = cancel? return from handler
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-6", // F no cancel-presence-msg, but if any cancel send empty presence msg
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-7", // F same but logs outgoing empty presence message. (msg is empty structure)
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-8", // crashes F? eventtimelogging
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-9", // crashes S? eventtimelogging + no cancel-presence-msg (should be slow)
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-10", // F eventtimelogging (should be fast)
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-11", // S eventtimelogging + no cancel-presence-msg (should be slow)
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-12", // F upload and download event logging (should be fast)
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-13", // S same but with no cancel-presence-msg (should be slow)
+
+            //"thatbenbierens/nim-codex:peerselecta-1", // F PR update (yes cancel-presence-msg)
+            //"thatbenbierens/nim-codex:peerselecta-2", // S PR update (no cancel-presence-msg)
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-14", // F new logging
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-15", // S new logging
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-16-f", // F more logging
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-16-s", // S more logging
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-17-f", // F "tick" every 100 milliseconds
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-17-s", // S same but slow
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-18-f", // F "tick" every 10 milliseconds
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-18-s", // S same but slow
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-19-f", // F sending/sent/received logs
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-19-s", // S same but slow
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-20-f", // F sending/sent/received logs + number
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-20-s", // S same but slow
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-21-f", // F libp2p lpchannel.write logs
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-21-s", // S same but slow
+
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-22-f", // F chronos stream write logs
+            //"thatbenbierens/nim-codex:blkex-cancelpresence-22-s", // S same but slow
+
+            "thatbenbierens/nim-codex:blkex-cancelpresence-23-f", // F chronos stream write logs in libp2p hand-off
+            "thatbenbierens/nim-codex:blkex-cancelpresence-23-s", // S same but slow
+
+            "thatbenbierens/nim-codex:blkex-cancelpresence-25-f", // F chronos stream write logs in libp2p hand-off with ticks
+            "thatbenbierens/nim-codex:blkex-cancelpresence-25-s",  // S same but slow
+
+            "thatbenbierens/nim-codex:blkex-cancelpresence-27-f", // F chronos stream write logs in libp2p hand-off with ticks adds names
+            "thatbenbierens/nim-codex:blkex-cancelpresence-27-s"  // S same but slow
+            )] string img
+        )
+        {
+            CodexContainerRecipe.DockerImageOverride = img;
+
+            var uploader = StartCodex(s => s.WithName("Uploader"));
+            var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
+
+            PerformTwoClientTest(uploader, downloader);
+        }
+
+        [Test]
+        public void ParseLogs()
+        {
+            var path = "d:\\Dev\\cs-codex-dist-tests\\Tests\\CodexReleaseTests\\bin\\Debug\\net8.0\\CodexTestLogs\\2025-01\\09\\13-58-28Z_TwoClientTests\\";
+            var file1 = Path.Combine(path, "TwoClientTest[thatbenbierens_nim-codex_blkex-cancelpresence-27-f]_000001_Downloader1.log");
+            var file2 = Path.Combine(path, "TwoClientTest[thatbenbierens_nim-codex_blkex-cancelpresence-27-f]_000000_Uploader0.log");
+            var file3 = Path.Combine(path, "TwoClientTest[thatbenbierens_nim-codex_blkex-cancelpresence-27-s]_000001_Downloader1.log");
+            var file4 = Path.Combine(path, "TwoClientTest[thatbenbierens_nim-codex_blkex-cancelpresence-27-s]_000000_Uploader0.log");
+
+            var lines = File.ReadAllLines(file3);
+            var clines = new List<CodexLogLine>();
+            foreach (var line in lines)
+            {
+                var cline = CodexLogLine.Parse(line);
+                if (cline != null) clines.Add(cline);
+            }
+
+            var gaps = new List<Gap>();
+            for (var i = 0; i < clines.Count; i++)
+            {
+                var line = clines[i];
+
+
+                // todo:
+//TRC 2025-01-09 13:59:14.501+00:00 chronosread                                topics="libp2p chronosstream custom" tid=1 ticks=424485 name=ChronosStream count=32669
+//TRC 2025-01-09 13:59:14.501+00:00 chronosread                                topics="libp2p chronosstream custom" tid=1 ticks=600 name=ChronosStream count=32670
+//TRC 2025-01-09 13:59:14.501+00:00 readOnce                                   topics="libp2p mplexchannel custom" tid=1 s=16U*uBBR7j:677fd62fe0c5bd152c675e42:677fd62ff7548faf70a27174 bytes=1 count=32671
+//TRC 2025-01-09 13:59:14.501+00:00 readOnce                                   topics="libp2p mplexchannel custom" tid=1 s=16U*uBBR7j:677fd62fe0c5bd152c675e42:677fd62ff7548faf70a27174 bytes=73 count=32672
+//TRC 2025-01-09 13:59:14.501+00:00 MsgReceived                                topics="codex blockexcnetworkpeer" tid=1 num=7 count=32673
+
+                // read to received!???
+
+                // run in cluster, same effect???
+                // run native, same effect?
+
+                if (line.Message == "MsgSending")
+                {
+                    // the next line is lpc-write-fast, then chronoswrite
+                    if (i + 2  < clines.Count)
+                    {
+                        var next = clines[i + 2];
+                        if (next.Message == "chronoswrite")
+                        {
+                            // got ya!
+                            gaps.Add(new Gap(line, next));
+                        }
+                        else
+                        {
+                            var aaaa = "what is it?!";
+                        }
+                    }
+                }
+            }
+
+            gaps = gaps.OrderByDescending(g => g.GapSpan.TotalMilliseconds).ToList();
+
+            var iiii = 0;
+
+        }
+
+        public class Gap
+        {
+            public Gap(CodexLogLine line, CodexLogLine next)
+            {
+                Line = line;
+                Next = next;
+            }
+
+            public CodexLogLine Line { get; }
+            public CodexLogLine Next { get; }
+
+            public TimeSpan GapSpan
+            {
+                get
+                {
+                    return Next.TimestampUtc - Line.TimestampUtc;
+                }
+            }
+
+            public override string ToString()
+            {
+                return $"[{GapSpan.TotalMilliseconds} ms]";
+            }
+        }
+
+        private void ProcessTimes(CodexLogLine cline)
+        {
+            // reqCreatedTime
+            // wantHaveSentTimes
+            // presenceRecvTimes
+            // wantBlkSentTimes
+            // blkRecvTimes
+            // cancelSentTimes
+            // resolveTimes
+
+        }
+
+        public class BlockReqTimes
+        {
+            public TimeSpan CreateToWantHaveSent { get; set; }
+
+        }
+
+        private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader)
+        {
+            PerformTwoClientTest(uploader, downloader, 100.MB());
+        }
+
+        private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader, ByteSize size)
+        {
+            var testFile = GenerateTestFile(size);
+
+            var contentId = uploader.UploadFile(testFile);
+            AssertNodesContainFile(contentId, uploader);
+
+            var (downloadedFile, timeTaken) = downloader.DownloadContentT(contentId);
+            AssertNodesContainFile(contentId, uploader, downloader);
+
+            Assert.That(timeTaken, Is.LessThan(TimeSpan.FromSeconds(15.0)), "Too slow!");
+
+            testFile.AssertIsEqual(downloadedFile);
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs

@@ -0,0 +1,38 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class UnknownCidTest : CodexDistTest
+    {
+        [Test]
+        public void DownloadingUnknownCidDoesNotCauseCrash()
+        {
+            var node = StartCodex();
+
+            var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ");
+
+            var localFiles = node.LocalFiles().Content;
+            CollectionAssert.DoesNotContain(localFiles.Select(f => f.Cid), unknownCid);
+
+            try
+            {
+                node.DownloadContent(unknownCid);
+            }
+            catch (Exception ex)
+            {
+                var expectedMessage = $"Download of '{unknownCid.Id}' timed out";
+                if (!ex.Message.StartsWith(expectedMessage)) throw;
+            }
+
+            WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), node);
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs

@@ -0,0 +1,111 @@
+using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    public class ContractFailedTest : MarketplaceAutoBootstrapDistTest
+    {
+        protected override int NumberOfHosts => 4;
+        protected override int NumberOfClients => 1;
+        protected override ByteSize HostAvailabilitySize => 1.GB();
+        protected override TimeSpan HostAvailabilityMaxDuration => TimeSpan.FromDays(1.0);
+        private readonly TestToken pricePerSlotPerSecond = 10.TstWei();
+
+        [Test]
+        public void ContractFailed()
+        {
+            var hosts = StartHosts();
+            var client = StartClients().Single();
+            StartValidator();
+
+            var request = CreateStorageRequest(client);
+
+            request.WaitForStorageContractSubmitted();
+            AssertContractIsOnChain(request);
+
+            request.WaitForStorageContractStarted();
+            AssertContractSlotsAreFilledByHosts(request, hosts);
+
+            hosts.BringOffline(waitTillStopped: true);
+
+            WaitForSlotFreedEvents();
+
+            request.WaitForContractFailed();
+        }
+
+        private void WaitForSlotFreedEvents()
+        {
+            Log(nameof(WaitForSlotFreedEvents));
+
+            var start = DateTime.UtcNow;
+            var timeout = CalculateContractFailTimespan();
+
+            while (DateTime.UtcNow < start + timeout)
+            {
+                var events = GetContracts().GetEvents(GetTestRunTimeRange());
+                var slotFreed = events.GetSlotFreedEvents();
+                if (slotFreed.Length == NumberOfHosts)
+                {
+                    Log($"{nameof(WaitForSlotFreedEvents)} took {Time.FormatDuration(DateTime.UtcNow - start)}");
+                    return;
+                }
+                GetContracts().WaitUntilNextPeriod();
+            }
+            Assert.Fail($"{nameof(WaitForSlotFreedEvents)} failed after {Time.FormatDuration(timeout)}");
+        }
+
+        private TimeSpan CalculateContractFailTimespan()
+        {
+            var config = GetContracts().Deployment.Config;
+            var maxSlashesBeforeSlotFreed = Convert.ToInt32(config.Collateral.MaxNumberOfSlashes);
+            var numProofsMissedBeforeSlash = Convert.ToInt32(config.Collateral.SlashCriterion);
+
+            var periodDuration = GetPeriodDuration();
+            var requiredNumMissedProofs = maxSlashesBeforeSlotFreed * numProofsMissedBeforeSlash;
+
+            // Each host could miss 1 proof per period,
+            // so the time we should wait is period time * requiredNum of missed proofs.
+            // Except: the proof requirement has a concept of "downtime":
+            // a segment of time where proof is not required.
+            // We calculate the probability of downtime and extend the waiting
+            // timeframe by a factor, such that all hosts are highly likely to have 
+            // failed a sufficient number of proofs.
+
+            float n = requiredNumMissedProofs;
+            return periodDuration * n * GetDowntimeFactor(config);
+        }
+
+        private float GetDowntimeFactor(MarketplaceConfig config)
+        {
+            byte numBlocksInDowntimeSegment = config.Proofs.Downtime;
+            float downtime = numBlocksInDowntimeSegment;
+            float window = 256.0f;
+            var chanceOfDowntime = downtime / window;
+            return 1.0f + chanceOfDowntime + chanceOfDowntime;
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
+        {
+            var cid = client.UploadFile(GenerateTestFile(5.MB()));
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = TimeSpan.FromHours(1.0),
+                Expiry = TimeSpan.FromHours(0.2),
+                MinRequiredNumberOfNodes = (uint)NumberOfHosts,
+                NodeFailureTolerance = (uint)(NumberOfHosts / 2),
+                PricePerSlotPerSecond = pricePerSlotPerSecond,
+                ProofProbability = 1, // Require a proof every period
+                RequiredCollateral = 1.Tst()
+            });
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/ContractRepairedTest.cs

@@ -0,0 +1,18 @@
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodexReleaseTests.MarketTests
+{
+    public class ContractRepairedTest
+    {
+        [Test]
+        [Ignore("TODO - Test in which a host fails, but the slot is repaired")]
+        public void ContractRepaired()
+        {
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs

@@ -0,0 +1,75 @@
+using CodexContractsPlugin;
+using CodexPlugin;
+using GethPlugin;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    [TestFixture]
+    public class ContractSuccessfulTest : MarketplaceAutoBootstrapDistTest
+    {
+        private const int FilesizeMb = 10;
+
+        protected override int NumberOfHosts => 6;
+        protected override int NumberOfClients => 1;
+        protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB();
+        protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration();
+        private readonly TestToken pricePerSlotPerSecond = 10.TstWei();
+
+        [Test]
+        public void ContractSuccessful()
+        {
+            var hosts = StartHosts();
+            var client = StartClients().Single();
+
+            var request = CreateStorageRequest(client);
+
+            request.WaitForStorageContractSubmitted();
+            AssertContractIsOnChain(request);
+
+            request.WaitForStorageContractStarted();
+            AssertContractSlotsAreFilledByHosts(request, hosts);
+
+            request.WaitForStorageContractFinished(GetContracts());
+
+            AssertClientHasPaidForContract(pricePerSlotPerSecond, client, request, hosts);
+            AssertHostsWerePaidForContract(pricePerSlotPerSecond, request, hosts);
+            AssertHostsCollateralsAreUnchanged(hosts);
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
+        {
+            var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
+            var config = GetContracts().Deployment.Config;
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = GetContractDuration(),
+                Expiry = GetContractExpiry(),
+                // TODO: this should work with NumberOfHosts, but
+                // an ongoing issue makes hosts sometimes not pick up slots.
+                // When it's resolved, we can reduce the number of hosts and slim down this test.
+                MinRequiredNumberOfNodes = 3,
+                NodeFailureTolerance = 1,
+                PricePerSlotPerSecond = pricePerSlotPerSecond,
+                ProofProbability = 20,
+                RequiredCollateral = 1.Tst()
+            });
+        }
+
+        private TimeSpan GetContractExpiry()
+        {
+            return GetContractDuration() / 2;
+        }
+
+        private TimeSpan GetContractDuration()
+        {
+            return Get8TimesConfiguredPeriodDuration() / 2;
+        }
+
+        private TimeSpan Get8TimesConfiguredPeriodDuration()
+        {
+            return GetPeriodDuration() * 8.0;
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs

@@ -0,0 +1,279 @@
+using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
+using CodexPlugin;
+using CodexTests;
+using DistTestCore;
+using GethPlugin;
+using Nethereum.Hex.HexConvertors.Extensions;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest
+    {
+        private readonly Dictionary<TestLifecycle, MarketplaceHandle> handles = new Dictionary<TestLifecycle, MarketplaceHandle>();
+        protected const int StartingBalanceTST = 1000;
+        protected const int StartingBalanceEth = 10;
+
+        protected override void LifecycleStart(TestLifecycle lifecycle)
+        {
+            base.LifecycleStart(lifecycle);
+            var geth = StartGethNode(s => s.IsMiner());
+            var contracts = Ci.StartCodexContracts(geth);
+            handles.Add(lifecycle, new MarketplaceHandle(geth, contracts));
+        }
+
+        protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result)
+        {
+            base.LifecycleStop(lifecycle, result);
+            handles.Remove(lifecycle);
+        }
+
+        protected IGethNode GetGeth()
+        {
+            return handles[Get()].Geth;
+        }
+
+        protected ICodexContracts GetContracts()
+        {
+            return handles[Get()].Contracts;
+        }
+
+        protected TimeSpan GetPeriodDuration()
+        {
+            var config = GetContracts().Deployment.Config;
+            return TimeSpan.FromSeconds(((double)config.Proofs.Period));
+        }
+
+        protected abstract int NumberOfHosts { get; }
+        protected abstract int NumberOfClients { get; }
+        protected abstract ByteSize HostAvailabilitySize { get; }
+        protected abstract TimeSpan HostAvailabilityMaxDuration { get; }
+
+        public ICodexNodeGroup StartHosts()
+        {
+            var hosts = StartCodex(NumberOfHosts, s => s
+                .WithName("host")
+                .EnableMarketplace(GetGeth(), GetContracts(), m => m
+                    .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
+                    .AsStorageNode()
+                )
+            );
+
+            var config = GetContracts().Deployment.Config;
+            foreach (var host in hosts)
+            {
+                Assert.That(GetTstBalance(host).TstWei, Is.EqualTo(StartingBalanceTST.Tst().TstWei));
+                Assert.That(GetEthBalance(host).Wei, Is.EqualTo(StartingBalanceEth.Eth().Wei));
+
+                host.Marketplace.MakeStorageAvailable(new CodexPlugin.StorageAvailability(
+                    totalSpace: HostAvailabilitySize,
+                    maxDuration: HostAvailabilityMaxDuration,
+                    minPriceForTotalSpace: 1.TstWei(),
+                    maxCollateral: 999999.Tst())
+                );
+            }
+            return hosts;
+        }
+
+        public TestToken GetTstBalance(ICodexNode node)
+        {
+            return GetContracts().GetTestTokenBalance(node);
+        }
+
+        public TestToken GetTstBalance(EthAddress address)
+        {
+            return GetContracts().GetTestTokenBalance(address);
+        }
+
+        public Ether GetEthBalance(ICodexNode node)
+        {
+            return GetGeth().GetEthBalance(node);
+        }
+
+        public Ether GetEthBalance(EthAddress address)
+        {
+            return GetGeth().GetEthBalance(address);
+        }
+
+        public ICodexNodeGroup StartClients()
+        {
+            return StartCodex(NumberOfClients, s => s
+                .WithName("client")
+                .EnableMarketplace(GetGeth(), GetContracts(), m => m
+                    .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
+                )
+            );
+        }
+
+        public ICodexNode StartValidator()
+        {
+            return StartCodex(s => s
+                .WithName("validator")
+                .EnableMarketplace(GetGeth(), GetContracts(), m => m
+                    .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
+                    .AsValidator()
+                )
+            );
+        }
+
+        public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts, string purchaseId)
+        {
+            var fills = GetOnChainSlotFills(possibleHosts);
+            return fills.Where(f => f
+                .SlotFilledEvent.RequestId.ToHex(false).ToLowerInvariant() == purchaseId.ToLowerInvariant())
+                .ToArray();
+        }
+
+        public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts)
+        {
+            var events = GetContracts().GetEvents(GetTestRunTimeRange());
+            var fills = events.GetSlotFilledEvents();
+            return fills.Select(f =>
+            {
+                var host = possibleHosts.Single(h => h.EthAddress.Address == f.Host.Address);
+                return new SlotFill(f, host);
+
+            }).ToArray();
+        }
+
+        protected void AssertClientHasPaidForContract(TestToken pricePerSlotPerSecond, ICodexNode client, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
+        {
+            var balance = GetTstBalance(client);
+            var expectedBalance = StartingBalanceTST.Tst() - GetContractFinalCost(pricePerSlotPerSecond, contract, hosts);
+
+            Assert.That(balance, Is.EqualTo(expectedBalance), "Client balance incorrect.");
+        }
+
+        protected void AssertHostsWerePaidForContract(TestToken pricePerSlotPerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
+        {
+            var fills = GetOnChainSlotFills(hosts);
+            var submitUtc = GetContractOnChainSubmittedUtc(contract);
+            var finishUtc = submitUtc + contract.Purchase.Duration;
+            var expectedBalances = new Dictionary<EthAddress, TestToken>();
+            foreach (var host in hosts) expectedBalances.Add(host.EthAddress, StartingBalanceTST.Tst());
+            foreach (var fill in fills)
+            {
+                var slotDuration = finishUtc - fill.SlotFilledEvent.Block.Utc;
+                expectedBalances[fill.Host.EthAddress] += GetContractCostPerSlot(pricePerSlotPerSecond, slotDuration);
+            }
+
+            foreach (var pair in expectedBalances)
+            {
+                var balance = GetTstBalance(pair.Key);
+                Assert.That(balance, Is.EqualTo(pair.Value), "Host was not paid for storage.");
+            }
+        }
+
+        protected void AssertHostsCollateralsAreUnchanged(ICodexNodeGroup hosts)
+        {
+            // There is no separate collateral location yet.
+            // All host balances should be equal to or greater than the starting balance.
+            foreach (var host in hosts)
+            {
+                Assert.That(GetTstBalance(host), Is.GreaterThanOrEqualTo(StartingBalanceTST.Tst()));
+            }
+        }
+
+        private TestToken GetContractFinalCost(TestToken pricePerSlotPerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
+        {
+            var fills = GetOnChainSlotFills(hosts);
+            var result = 0.Tst();
+            var submitUtc = GetContractOnChainSubmittedUtc(contract);
+            var finishUtc = submitUtc + contract.Purchase.Duration;
+
+            foreach (var fill in fills)
+            {
+                var slotDuration = finishUtc - fill.SlotFilledEvent.Block.Utc;
+                result += GetContractCostPerSlot(pricePerSlotPerSecond, slotDuration);
+            }
+
+            return result;
+        }
+
+        private DateTime GetContractOnChainSubmittedUtc(IStoragePurchaseContract contract)
+        {
+            return Time.Retry<DateTime>(() =>
+            {
+                var events = GetContracts().GetEvents(GetTestRunTimeRange());
+                var submitEvent = events.GetStorageRequests().SingleOrDefault(e => e.RequestId.ToHex(false) == contract.PurchaseId);
+                if (submitEvent == null)
+                {
+                    // We're too early.
+                    throw new TimeoutException(nameof(GetContractOnChainSubmittedUtc) + "StorageRequest not found on-chain.");
+                }
+                return submitEvent.Block.Utc;
+            }, nameof(GetContractOnChainSubmittedUtc));
+        }
+
+        private TestToken GetContractCostPerSlot(TestToken pricePerSlotPerSecond, TimeSpan slotDuration)
+        {
+            return pricePerSlotPerSecond * (int)slotDuration.TotalSeconds;
+        }
+
+        protected void AssertContractSlotsAreFilledByHosts(IStoragePurchaseContract contract, ICodexNodeGroup hosts)
+        {
+            var activeHosts = new Dictionary<int, SlotFill>();
+
+            Time.Retry(() =>
+            {
+                var fills = GetOnChainSlotFills(hosts, contract.PurchaseId);
+                foreach (var fill in fills)
+                {
+                    var index = (int)fill.SlotFilledEvent.SlotIndex;
+                    if (!activeHosts.ContainsKey(index))
+                    {
+                        activeHosts.Add(index, fill);
+                    }
+                }
+
+                if (activeHosts.Count != contract.Purchase.MinRequiredNumberOfNodes) throw new Exception("Not all slots were filled...");
+
+            }, nameof(AssertContractSlotsAreFilledByHosts));
+        }
+
+        protected void AssertContractIsOnChain(IStoragePurchaseContract contract)
+        {
+            AssertOnChainEvents(events =>
+            {
+                var onChainRequests = events.GetStorageRequests();
+                if (onChainRequests.Any(r => r.Id == contract.PurchaseId)) return;
+                throw new Exception($"OnChain request {contract.PurchaseId} not found...");
+            }, nameof(AssertContractIsOnChain));
+        }
+
+        protected void AssertOnChainEvents(Action<ICodexContractsEvents> onEvents, string description)
+        {
+            Time.Retry(() =>
+            {
+                var events = GetContracts().GetEvents(GetTestRunTimeRange());
+                onEvents(events);
+            }, description);
+        }
+
+        public class SlotFill
+        {
+            public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host)
+            {
+                SlotFilledEvent = slotFilledEvent;
+                Host = host;
+            }
+
+            public SlotFilledEventDTO SlotFilledEvent { get; }
+            public ICodexNode Host { get; }
+        }
+
+        private class MarketplaceHandle
+        {
+            public MarketplaceHandle(IGethNode geth, ICodexContracts contracts)
+            {
+                Geth = geth;
+                Contracts = contracts;
+            }
+
+            public IGethNode Geth { get; }
+            public ICodexContracts Contracts { get; }
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/MultipleContractsTest.cs

@@ -0,0 +1,83 @@
+using CodexContractsPlugin;
+using CodexPlugin;
+using GethPlugin;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    [TestFixture]
+    public class MultipleContractsTest : MarketplaceAutoBootstrapDistTest
+    {
+        private const int FilesizeMb = 10;
+
+        protected override int NumberOfHosts => 8;
+        protected override int NumberOfClients => 3;
+        protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB();
+        protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration();
+        private readonly TestToken pricePerSlotPerSecond = 10.TstWei();
+
+        [Test]
+        [Ignore("TODO - Test where multiple successful contracts are run simultaenously")]
+        public void MultipleSuccessfulContracts()
+        {
+            var hosts = StartHosts();
+            var clients = StartClients();
+
+            var requests = clients.Select(c => CreateStorageRequest(c)).ToArray();
+
+            All(requests, r =>
+            {
+                r.WaitForStorageContractSubmitted();
+                AssertContractIsOnChain(r);
+            });
+
+            All(requests, r => r.WaitForStorageContractStarted());
+            All(requests, r => AssertContractSlotsAreFilledByHosts(r, hosts));
+
+            All(requests, r => r.WaitForStorageContractFinished(GetContracts()));
+
+            // todo:
+            //AssertClientHasPaidForContract(pricePerSlotPerSecond, client, request, hosts);
+            //AssertHostsWerePaidForContract(pricePerSlotPerSecond, request, hosts);
+            //AssertHostsCollateralsAreUnchanged(hosts);
+        }
+
+        private void All(IStoragePurchaseContract[] requests, Action<IStoragePurchaseContract> action)
+        {
+            foreach (var r in requests) action(r);
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
+        {
+            var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
+            var config = GetContracts().Deployment.Config;
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = GetContractDuration(),
+                Expiry = GetContractExpiry(),
+                MinRequiredNumberOfNodes = (uint)NumberOfHosts,
+                NodeFailureTolerance = (uint)(NumberOfHosts / 2),
+                PricePerSlotPerSecond = pricePerSlotPerSecond,
+                ProofProbability = 20,
+                RequiredCollateral = 1.Tst()
+            });
+        }
+
+        private TimeSpan GetContractExpiry()
+        {
+            return GetContractDuration() / 2;
+        }
+
+        private TimeSpan GetContractDuration()
+        {
+            return Get8TimesConfiguredPeriodDuration() / 2;
+        }
+
+        private TimeSpan Get8TimesConfiguredPeriodDuration()
+        {
+            var config = GetContracts().Deployment.Config;
+            return TimeSpan.FromSeconds(((double)config.Proofs.Period) * 8.0);
+        }
+    }
+}

Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs

@@ -0,0 +1,62 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexReleaseTests.NodeTests
+{
+    [TestFixture]
+    public class BasicInfoTests : CodexDistTest
+    {
+        [Test]
+        public void QuotaTest()
+        {
+            var size = 3.GB();
+            var node = StartCodex(s => s.WithStorageQuota(size));
+            var space = node.Space();
+
+            Assert.That(space.QuotaMaxBytes, Is.EqualTo(size.SizeInBytes));
+        }
+
+        [Test]
+        public void Spr()
+        {
+            var node = StartCodex();
+            
+            var info = node.GetDebugInfo();
+            Assert.That(!string.IsNullOrEmpty(info.Spr));
+
+            var spr = node.GetSpr();
+            Assert.That(!string.IsNullOrEmpty(spr));
+
+            Assert.That(info.Spr, Is.EqualTo(spr));
+        }
+
+        [Test]
+        public void VersionInfo()
+        {
+            var node = StartCodex();
+
+            var info = node.GetDebugInfo();
+            Assert.That(!string.IsNullOrEmpty(info.Version.Version));
+            Assert.That(!string.IsNullOrEmpty(info.Version.Revision));
+        }
+
+        [Test]
+        public void AnnounceAddress()
+        {
+            var node = StartCodex();
+            var addr = node.Container.GetInternalAddress(CodexContainerRecipe.ListenPortTag);
+
+            var info = node.GetDebugInfo();
+
+            Assert.That(info.AnnounceAddresses.Count, Is.GreaterThan(0));
+            // Ideally we'd assert the pod IP is in the announce address, but we can't access it from here.
+        }
+    }
+}

Tests/CodexReleaseTests/NodeTests/PeerTableTests.cs

@@ -0,0 +1,31 @@
+using CodexPlugin;
+using CodexTests;
+using CodexTests.Helpers;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexReleaseTests.NodeTests
+{
+    [TestFixture]
+    public class PeerTableTests : AutoBootstrapDistTest
+    {
+        [Test]
+        public void PeerTableCompleteness()
+        {
+            var nodes = StartCodex(10);
+
+            AssertAllNodesSeeEachOther(nodes.Concat([BootstrapNode!]));
+        }
+
+        private void AssertAllNodesSeeEachOther(IEnumerable<ICodexNode> nodes)
+        {
+            var helper = new PeerConnectionTestHelpers(GetTestLog());
+            helper.AssertFullyConnected(nodes);
+        }
+    }
+}

Tests/CodexReleaseTests/Parallelism.cs

@@ -0,0 +1,6 @@
+using NUnit.Framework;
+
+[assembly: LevelOfParallelism(1)]
+namespace CodexReleaseTests.DataTests
+{
+}

Tests/CodexTests/BasicTests/EventLogginHandler.cs

@@ -1,57 +0,0 @@
-using CodexContractsPlugin.ChainMonitor;
-using GethPlugin;
-using Logging;
-using System.Numerics;
-
-namespace CodexTests.BasicTests
-{
-    public class EventLogginHandler : IChainStateChangeHandler
-    {
-        private readonly ILog log;
-
-        public EventLogginHandler(ILog log)
-        {
-            this.log = log;
-        }
-
-        public void OnNewRequest(RequestEvent requestEvent)
-        {
-            Log(nameof(OnNewRequest), requestEvent);
-        }
-
-        public void OnRequestCancelled(RequestEvent requestEvent)
-        {
-            Log(nameof(OnRequestCancelled), requestEvent);
-        }
-
-        public void OnRequestFailed(RequestEvent requestEvent)
-        {
-            Log(nameof(OnRequestFailed), requestEvent);
-        }
-
-        public void OnRequestFinished(RequestEvent requestEvent)
-        {
-            Log(nameof(OnRequestFinished), requestEvent);
-        }
-
-        public void OnRequestFulfilled(RequestEvent requestEvent)
-        {
-            Log(nameof(OnRequestFulfilled), requestEvent);
-        }
-
-        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
-        {
-            Log(nameof(OnSlotFilled), requestEvent, host.ToString(), slotIndex.ToString());
-        }
-
-        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
-        {
-            Log(nameof(OnNewRequest), requestEvent, slotIndex.ToString());
-        }
-
-        private void Log(string name, object o, params string[] str)
-        {
-            log.Log(name + ": " + o.ToString() + " - " + string.Join(",", str));
-        }
-    }
-}

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -1,74 +0,0 @@
-using CodexPlugin;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.BasicTests
-{
-    [TestFixture]
-    public class ThreeClientTest : AutoBootstrapDistTest
-    {
-        [Test]
-        public void ThreeClient()
-        {
-            var primary = StartCodex();
-            var secondary = StartCodex();
-
-            var testFile = GenerateTestFile(10.MB());
-
-            var contentId = primary.UploadFile(testFile);
-
-            var downloadedFile = secondary.DownloadContent(contentId);
-
-            testFile.AssertIsEqual(downloadedFile);
-        }
-
-        [Test]
-        public void FindBug()
-        {
-            var uploader = StartCodex();
-            var downloaders = StartCodex(10);
-
-            var start = DateTime.UtcNow;
-            while ((DateTime.UtcNow - start) < TimeSpan.FromMinutes(15))
-            {
-                var cid = uploader.UploadFile(GenerateTestFile(5.MB()));
-
-                var loop = Parallel.ForEach(downloaders, d =>
-                {
-                    d.DownloadContent(cid);
-                });
-
-                Assert.That(loop.IsCompleted);
-            }
-        }
-
-        [Test]
-        public void DownloadingUnknownCidDoesNotCauseCrash()
-        {
-            var node = StartCodex(2).First();
-
-            var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ");
-
-            try
-            {
-                node.DownloadContent(unknownCid);
-            }
-            catch (Exception ex)
-            {
-                if (!ex.Message.StartsWith("Retry 'DownloadFile' timed out"))
-                {
-                    throw;
-                }
-            }
-
-            // Check that the node stays alive for at least another 5 minutes.
-            var start = DateTime.UtcNow;
-            while ((DateTime.UtcNow - start) < TimeSpan.FromMinutes(5))
-            {
-                Thread.Sleep(5000);
-                var info = node.GetDebugInfo();
-                Assert.That(!string.IsNullOrEmpty(info.Id));
-            }
-        }
-    }
-}

Tests/CodexTests/BasicTests/TwoClientTests.cs

@@ -1,52 +0,0 @@
-using CodexPlugin;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.BasicTests
-{
-    [TestFixture]
-    public class TwoClientTests : CodexDistTest
-    {
-        [Test]
-        public void TwoClientTest()
-        {
-            var uploader = StartCodex(s => s.WithName("Uploader"));
-            var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
-
-            PerformTwoClientTest(uploader, downloader);
-        }
-
-        [Test]
-        public void TwoClientsTwoLocationsTest()
-        {
-            var locations = Ci.GetKnownLocations();
-            if (locations.NumberOfLocations < 2)
-            {
-                Assert.Inconclusive("Two-locations test requires 2 nodes to be available in the cluster.");
-                return;
-            }
-
-            var uploader = Ci.StartCodexNode(s => s.WithName("Uploader").At(locations.Get(0)));
-            var downloader = Ci.StartCodexNode(s => s.WithName("Downloader").WithBootstrapNode(uploader).At(locations.Get(1)));
-
-            PerformTwoClientTest(uploader, downloader);
-        }
-
-        private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader)
-        {
-            PerformTwoClientTest(uploader, downloader, 10.MB());
-        }
-
-        private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader, ByteSize size)
-        {
-            var testFile = GenerateTestFile(size);
-
-            var contentId = uploader.UploadFile(testFile);
-
-            var downloadedFile = downloader.DownloadContent(contentId);
-
-            testFile.AssertIsEqual(downloadedFile);
-            CheckLogForErrors(uploader, downloader);
-        }
-    }
-}

Tests/DistTestCore/Configuration.cs

@@ -29,12 +29,12 @@ namespace DistTestCore
         /// </summary>
         public bool AlwaysDownloadContainerLogs { get; set; }
 
-        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace, Func<string?, string?> replacer)
+        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace)
         {
-            return GetK8sConfiguration(timeSet, new DoNothingK8sHooks(), k8sNamespace, replacer);
+            return GetK8sConfiguration(timeSet, new DoNothingK8sHooks(), k8sNamespace);
         }
 
-        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, IK8sHooks hooks, string k8sNamespace, Func<string?, string?> replacer)
+        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, IK8sHooks hooks, string k8sNamespace)
         {
             var config = new KubernetesWorkflow.Configuration(
                 kubeConfigFile: kubeConfigFile,
@@ -45,7 +45,6 @@ namespace DistTestCore
 
             config.AllowNamespaceOverride = false;
             config.Hooks = hooks;
-            config.Replacer = replacer;
 
             return config;
         }

Tests/DistTestCore/DistTest.cs

@@ -35,7 +35,7 @@ namespace DistTestCore
             fixtureLog = new FixtureLog(logConfig, startTime, deployId);
             statusLog = new StatusLog(logConfig, startTime, "dist-tests", deployId);
 
-            globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix, s => s), configuration.GetFileManagerFolder());
+            globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder());
 
             Initialize(fixtureLog);
         }

Tests/DistTestCore/DownloadedLogExtensions.cs

@@ -23,5 +23,21 @@ namespace DistTestCore
             }
             CollectionAssert.IsEmpty(errors);
         }
+
+        public static void AssertLogDoesNotContainLinesStartingWith(this IDownloadedLog log, params string[] unexpectedStrings)
+        {
+            var errors = new List<string>();
+            log.IterateLines(line =>
+            {
+                foreach (var str in unexpectedStrings)
+                {
+                    if (line.StartsWith(str))
+                    {
+                        errors.Add($"Found '{str}' at start of line '{line}'.");
+                    }
+                }
+            });
+            CollectionAssert.IsEmpty(errors);
+        }
     }
 }