Sets up tests for symmetric, asymmetric and binary speed checks

* Fix job status check

* Add a delay before get Kubernetes job status

.gitattributes

@@ -0,0 +1,10 @@
+# Set default behavior to automatically normalize line endings.
+* text=auto
+
+# Force bash scripts to always use lf line endings so that if a repo is accessed
+# in Unix via a file share from Windows, the scripts will work.
+*.sh text eol=lf
+
+# Likewise, force cmd and batch scripts to always use crlf
+*.cmd text eol=crlf
+*.bat text eol=crlf

.github/workflows/run-continuous-tests.yaml

@@ -80,7 +80,7 @@ env:
   TESTS_TARGET_DURATION: 2d
   TESTS_FILTER: ""
   TESTS_CLEANUP: true
-  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  JOB_MANIFEST: docker/job-continuous-tests.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
   KUBE_VERSION: v1.28.2
 
@@ -146,7 +146,7 @@ jobs:
         if: false
         run: |
           # Variables
-          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to run a node
           duration=600
           namespace="${{ env.NAMESPACE }}"
           pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")

.github/workflows/run-dist-tests.yaml

@@ -32,7 +32,7 @@ env:
   NAMEPREFIX: d-tests-runner
   NAMESPACE: default
   COMMAND: dotnet test Tests/CodexTests
-  JOB_MANIFEST: docker/dist-tests-job.yaml
+  JOB_MANIFEST: docker/job-dist-tests.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
   KUBE_VERSION: v1.28.2
 

.github/workflows/run-release-tests.yaml

@@ -0,0 +1,122 @@
+name: Run Release Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+      workflow_source:
+        description: Workflow source
+        required: false
+        type: string
+        default: ''
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
+  TEST_TYPE: release-tests
+  NAMEPREFIX: r-tests
+  NAMESPACE: default
+  JOB_MANIFEST: docker/job-release-tests.yaml
+  COMMAND: dotnet test Tests/CodexReleaseTests
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.30.5
+
+
+jobs:
+  run_tests:
+    name: Run Release Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
+          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'${COMMAND}'"')" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job to run tests
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}"
+          echo "Runner pod: $(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}')"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+
+      - name: Show Runner logs
+        run: |
+          # Variables
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          namespace="${{ env.NAMESPACE }}"
+          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath='{.items[0].metadata.name}')
+
+          # Check Pod status
+          WAIT=120
+          SECONDS=0
+          sleep=1
+          while (( SECONDS < WAIT )); do
+            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}')
+            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in the $phase state - Get the logs"; break; } || { echo "Pod $pod is in the $phase state - Retry in $sleep second(s) / $((WAIT - SECONDS))"; }
+            sleep $sleep
+          done
+
+          # Get logs
+          while [[ $(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}') == "Running" ]]; do
+            echo "Show ${pod} logs ..."
+            echo "----"
+            kubectl logs $pod -n $namespace -f || true
+            sleep 1
+          done
+
+      - name: Set workflow status from job status
+        run: |
+          sleep 5
+          job_status=$(kubectl get jobs ${{ env.NAMEPREFIX }} -n ${{ env.NAMESPACE }} -o jsonpath='{.status.conditions[0].type}')
+          echo "Job status: $job_status"
+          if [[ "${job_status}" != "Complete" ]]; then exit 1; fi

Framework/ArgsUniform/ArgsUniform.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/BlockchainUtils/BlockCache.cs

@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockCache
     {

Framework/BlockchainUtils/BlockTimeEntry.cs

@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockTimeEntry
     {

Framework/BlockchainUtils/BlockTimeFinder.cs

@@ -1,6 +1,6 @@
 using Logging;
 
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockTimeFinder
     {
@@ -29,7 +29,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();
-            if (moment <= bounds.Genesis.Utc) return null;
+            if (moment < bounds.Genesis.Utc) return null;
+            if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
             if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
 
             return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
@@ -38,7 +39,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetLowestBlockNumberAfter(DateTime moment)
         {
             bounds.Initialize();
-            if (moment >= bounds.Current.Utc) return null;
+            if (moment > bounds.Current.Utc) return null;
+            if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber;
             if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
 
             return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;

Framework/BlockchainUtils/BlockchainBounds.cs

@@ -1,5 +1,11 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
+    public interface IWeb3Blocks
+    {
+        ulong GetCurrentBlockNumber();
+        DateTime? GetTimestampForBlock(ulong blockNumber);
+    }
+
     public class BlockchainBounds
     {
         private readonly BlockCache cache;

Framework/BlockchainUtils/BlockchainUtils.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+	<ItemGroup>
+		<PackageReference Include="Nethereum.Web3" Version="4.14.0" />
+	</ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/BlockchainUtils/ConversionExtensions.cs

@@ -1,7 +1,7 @@
 using Nethereum.Hex.HexTypes;
 using System.Numerics;
 
-namespace NethereumWorkflow
+namespace BlockchainUtils
 {
     public static class ConversionExtensions
     {

Framework/Core/Core.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/DiscordRewards/DiscordRewards.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -3,7 +3,8 @@
     public class GiveRewardsCommand
     {
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
-        public string[] EventsOverview { get; set; } = Array.Empty<string>();
+        public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty<ChainEventMessage>();
+        public string[] Errors { get; set; } = Array.Empty<string>();
 
         public bool HasAny()
         {
@@ -16,4 +17,10 @@
         public ulong RewardId { get; set; }
         public string[] UserAddresses { get; set; } = Array.Empty<string>();
     }
+
+    public class ChainEventMessage
+    {
+        public ulong BlockNumber { get; set; }
+        public string Message { get; set; } = string.Empty;
+    }
 }

Framework/FileUtils/FileUtils.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/GethConnector/GethConnector.cs

@@ -1,4 +1,6 @@
-using CodexContractsPlugin;
+using BlockchainUtils;
+using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 
@@ -18,18 +20,29 @@ namespace GethConnector
                 return null;
             }
 
+            var gethNode = new CustomGethNode(log, new BlockCache(), GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+
+            var config = GetCodexMarketplaceConfig(gethNode, GethInput.MarketplaceAddress);
+
             var contractsDeployment = new CodexContractsDeployment(
+                config: config,
                 marketplaceAddress: GethInput.MarketplaceAddress,
                 abi: GethInput.ABI,
                 tokenAddress: GethInput.TokenAddress
             );
 
-            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
             var contracts = new CodexContractsAccess(log, gethNode, contractsDeployment);
 
             return new GethConnector(gethNode, contracts);
         }
 
+        private static MarketplaceConfig GetCodexMarketplaceConfig(IGethNode gethNode, string marketplaceAddress)
+        {
+            var func = new ConfigurationFunctionBase();
+            var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
+            return response.ReturnValue1;
+        }
+
         private GethConnector(IGethNode gethNode, ICodexContracts codexContracts)
         {
             GethNode = gethNode;

Framework/GethConnector/GethConnector.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -91,7 +91,7 @@ namespace KubernetesWorkflow
         private void DownloadCrashedContainerLogs(Kubernetes client)
         {
             using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
-            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
+            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName, containerName);
             handler.Log(stream);
         }
     }

Framework/KubernetesWorkflow/DownloadedLog.cs

@@ -6,6 +6,7 @@ namespace KubernetesWorkflow
     {
         string ContainerName { get; }
 
+        void IterateLines(Action<string> action);
         void IterateLines(Action<string> action, params string[] thatContain);
         string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
@@ -25,58 +26,39 @@ namespace KubernetesWorkflow
 
         public string ContainerName { get; }
 
-        public void IterateLines(Action<string> action, params string[] thatContain)
+        public void IterateLines(Action<string> action)
         {
             using var file = File.OpenRead(logFile.FullFilename);
             using var streamReader = new StreamReader(file);
 
             var line = streamReader.ReadLine();
             while (line != null)
+            {
+                action(line);
+                line = streamReader.ReadLine();
+            }
+        }
+
+        public void IterateLines(Action<string> action, params string[] thatContain)
+        {
+            IterateLines(line =>
             {
                 if (thatContain.All(line.Contains))
                 {
                     action(line);
                 }
-                line = streamReader.ReadLine();
-            }
+            });
         }
 
         public string[] GetLinesContaining(string expectedString)
         {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-            var lines = new List<string>();
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (line.Contains(expectedString))
-                {
-                    lines.Add(line);
-                }
-                line = streamReader.ReadLine();
-            }
-
-            return lines.ToArray(); ;
+            return FindLinesThatContain([expectedString]);
         }
 
         public string[] FindLinesThatContain(params string[] tags)
         {
             var result = new List<string>();
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (tags.All(line.Contains))
-                {
-                    result.Add(line);
-                }
-
-                line = streamReader.ReadLine();
-            }
-
+            IterateLines(result.Add, tags);
             return result.ToArray();
         }
 

Framework/KubernetesWorkflow/K8sController.cs

@@ -906,7 +906,7 @@ namespace KubernetesWorkflow
                 var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
                 log.Error(msg);
 
-                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg, deploymentName), tailLines: null, previous: true);
 
                 throw new Exception(msg);
             }

Framework/KubernetesWorkflow/K8sNameUtils.cs

@@ -24,8 +24,8 @@
                 .Replace("]", "-")
                 .Replace(",", "-");
 
-            result = result.Trim('-');
             if (result.Length > maxLength) result = result.Substring(0, maxLength);
+            result = result.Trim('-');
 
             return result;
         }

Framework/KubernetesWorkflow/KubernetesWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>KubernetesWorkflow</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>

Framework/KubernetesWorkflow/LogHandler.cs

@@ -25,9 +25,9 @@ namespace KubernetesWorkflow
 
     public class WriteToFileLogHandler : LogHandler, ILogHandler
     {
-        public WriteToFileLogHandler(ILog sourceLog, string description)
+        public WriteToFileLogHandler(ILog sourceLog, string description, string addFileName)
         {
-            LogFile = sourceLog.CreateSubfile();
+            LogFile = sourceLog.CreateSubfile(addFileName);
 
             var msg = $"{description} -->> {LogFile.FullFilename}";
             sourceLog.Log(msg);

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -127,7 +127,7 @@ namespace KubernetesWorkflow
         {
             var msg = $"Downloading container log for '{container.Name}'";
             log.Log(msg);
-            var logHandler = new WriteToFileLogHandler(log, msg);
+            var logHandler = new WriteToFileLogHandler(log, msg, container.Name);
 
             K8s(controller =>
             {

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -24,13 +24,12 @@ namespace KubernetesWorkflow.Types
         [JsonIgnore]
         public RunningPod RunningPod { get; internal set; } = null!;
 
-        public Address GetAddress(ILog log, string portTag)
+        public Address GetAddress(string portTag)
         {
             var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
             if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);
 
             var select = SelectAddress(addresses);
-            log.Debug($"Container '{Name}' selected for tag '{portTag}' address: '{select}'");
             return select.Address;
         }
 

Framework/Logging/BaseLog.cs

@@ -8,7 +8,7 @@ namespace Logging
         void Debug(string message = "", int skipFrames = 0);
         void Error(string message);
         void AddStringReplace(string from, string to);
-        LogFile CreateSubfile(string ext = "log");
+        LogFile CreateSubfile(string addName, string ext = "log");
     }
 
     public abstract class BaseLog : ILog
@@ -72,9 +72,13 @@ namespace Logging
             File.Delete(LogFile.FullFilename);
         }
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
+            addName = addName
+                .Replace("<", "")
+                .Replace(">", "");
+
+            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}_{addName}", ext);
         }
 
         protected string ApplyReplacements(string str)

Framework/Logging/LogFile.cs

@@ -1,4 +1,6 @@
-namespace Logging
+using Utils;
+
+namespace Logging
 {
     public class LogFile
     {
@@ -49,7 +51,7 @@
 
         private static string GetTimestamp()
         {
-            return $"[{DateTime.UtcNow.ToString("o")}]";
+            return $"[{Time.FormatTimestamp(DateTime.UtcNow)}]";
         }
 
         private void EnsurePathExists(string filename)

Framework/Logging/LogPrefixer.cs

@@ -18,9 +18,9 @@
         public string Prefix { get; set; } = string.Empty;
 
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return backingLog.CreateSubfile(ext);
+            return backingLog.CreateSubfile(addName, ext);
         }
 
         public void Debug(string message = "", int skipFrames = 0)

Framework/Logging/LogSplitter.cs

@@ -14,9 +14,9 @@
             OnAll(l => l.AddStringReplace(from, to));
         }
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return targetLogs.First().CreateSubfile(ext);
+            return targetLogs.First().CreateSubfile(addName, ext);
         }
 
         public void Debug(string message = "", int skipFrames = 0)

Framework/Logging/Logging.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Logging</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -1,25 +1,25 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace NethereumWorkflow
 {
     public class NethereumInteraction
     {
-        // BlockCache is a static instance: It stays alive for the duration of the application runtime.
-        private readonly static BlockCache blockCache = new BlockCache();
+        private readonly BlockCache blockCache;
 
         private readonly ILog log;
         private readonly Web3 web3;
 
-        internal NethereumInteraction(ILog log, Web3 web3)
+        internal NethereumInteraction(ILog log, Web3 web3, BlockCache blockCache)
         {
             this.log = log;
             this.web3 = web3;
+            this.blockCache = blockCache;
         }
 
         public string SendEth(string toAddress, decimal ethAmount)

Framework/NethereumWorkflow/NethereumInteractionCreator.cs

@@ -1,4 +1,5 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.Web3;
 
 namespace NethereumWorkflow
@@ -6,13 +7,15 @@ namespace NethereumWorkflow
     public class NethereumInteractionCreator
     {
         private readonly ILog log;
+        private readonly BlockCache blockCache;
         private readonly string ip;
         private readonly int port;
         private readonly string privateKey;
 
-        public NethereumInteractionCreator(ILog log, string ip, int port, string privateKey)
+        public NethereumInteractionCreator(ILog log, BlockCache blockCache, string ip, int port, string privateKey)
         {
             this.log = log;
+            this.blockCache = blockCache;
             this.ip = ip;
             this.port = port;
             this.privateKey = privateKey;
@@ -21,7 +24,7 @@ namespace NethereumWorkflow
         public NethereumInteraction CreateWorkflow()
         {
             log.Debug("Starting interaction to " + ip + ":" + port);
-            return new NethereumInteraction(log, CreateWeb3());
+            return new NethereumInteraction(log, CreateWeb3(), blockCache);
         }
 
         private Web3 CreateWeb3()

Framework/NethereumWorkflow/NethereumWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	<RootNamespace>NethereumWorkflow</RootNamespace>
 	<ImplicitUsings>enable</ImplicitUsings>
 	<Nullable>enable</Nullable>
@@ -12,6 +12,7 @@
 	</ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\BlockchainUtils\BlockchainUtils.csproj" />
     <ProjectReference Include="..\Logging\Logging.csproj" />
     <ProjectReference Include="..\Utils\Utils.csproj" />
   </ItemGroup>

Framework/NethereumWorkflow/Web3Wrapper.cs

@@ -1,16 +1,11 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
 using Utils;
 
 namespace NethereumWorkflow
 {
-    public interface IWeb3Blocks
-    {
-        ulong GetCurrentBlockNumber();
-        DateTime? GetTimestampForBlock(ulong blockNumber);
-    }
-
     public class Web3Wrapper : IWeb3Blocks
     {
         private readonly Web3 web3;

Framework/OverwatchTranscript/EventBucketReader.cs

@@ -7,6 +7,7 @@ namespace OverwatchTranscript
     public interface IFinalizedBucket
     {
         bool IsEmpty { get; }
+        void Update();
         DateTime? SeeTopUtc();
         BucketTop? TakeTop();
     }
@@ -28,7 +29,8 @@ namespace OverwatchTranscript
         private readonly string bucketFile;
         private readonly ConcurrentQueue<BucketTop> topQueue = new ConcurrentQueue<BucketTop>();
         private readonly AutoResetEvent itemDequeued = new AutoResetEvent(false);
-        private bool stopping;
+        private readonly AutoResetEvent itemEnqueued = new AutoResetEvent(false);
+        private bool sourceIsEmpty;
 
         public EventBucketReader(ILog log, string bucketFile)
         {
@@ -42,34 +44,38 @@ namespace OverwatchTranscript
         
         public bool IsEmpty { get; private set; }
 
+        public void Update()
+        {
+            if (IsEmpty) return;
+            while (topQueue.Count == 0)
+            {
+                UpdateIsEmpty();
+                if (IsEmpty) return;
+
+                itemDequeued.Set();
+                itemEnqueued.WaitOne(200);
+            }
+        }
+
         public DateTime? SeeTopUtc()
         {
             if (IsEmpty) return null;
-            while (true)
+            if (topQueue.TryPeek(out BucketTop? top))
             {
-                UpdateIsEmpty();
-                if (IsEmpty) return null;
-                if (topQueue.TryPeek(out BucketTop? top))
-                {
-                    return top.Utc;
-                }
+                return top.Utc;
             }
+            return null;
         }
 
         public BucketTop? TakeTop()
         {
             if (IsEmpty) return null;
-
-            while (true)
+            if (topQueue.TryDequeue(out BucketTop? top))
             {
-                UpdateIsEmpty();
-                if (IsEmpty) return null;
-                if (topQueue.TryDequeue(out BucketTop? top))
-                {
-                    itemDequeued.Set();
-                    return top;
-                }
+                itemDequeued.Set();
+                return top;
             }
+            return null;
         }
 
         private void ReadBucket()
@@ -85,23 +91,25 @@ namespace OverwatchTranscript
                     if (top != null)
                     {
                         topQueue.Enqueue(top);
+                        itemEnqueued.Set();
                     }
                     else
                     {
-                        stopping = true;
+                        sourceIsEmpty = true;
+                        UpdateIsEmpty();
                         return;
                     }
                 }
 
                 itemDequeued.Reset();
-                itemDequeued.WaitOne();
+                itemDequeued.WaitOne(5000);
             }
         }
 
         private void UpdateIsEmpty()
         {
-            var empty = stopping && topQueue.IsEmpty;
-            if (!IsEmpty && empty)
+            var allEmpty = sourceIsEmpty && topQueue.IsEmpty;
+            if (!IsEmpty && allEmpty)
             {
                 File.Delete(bucketFile);
                 IsEmpty = true;

Framework/OverwatchTranscript/MomentReferenceBuilder.cs

@@ -24,6 +24,8 @@ namespace OverwatchTranscript
             log.Debug($"Building references for {buckets.Count} buckets.");
             while (buckets.Any())
             {
+                foreach (var b in buckets) b.Update();
+
                 buckets.RemoveAll(b => b.IsEmpty);
                 if (!buckets.Any()) break;
 

Framework/OverwatchTranscript/OverwatchTranscript.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/Utils/BlockInterval.cs

@@ -15,12 +15,13 @@
                 To = from;
             }
             TimeRange = timeRange;
+            NumberOfBlocks = (To - From) + 1;
         }
 
         public ulong From { get; }
         public ulong To { get; }
         public TimeRange TimeRange { get; }
-        public ulong NumberOfBlocks => To - From;
+        public ulong NumberOfBlocks { get; }
 
         public override string ToString()
         {

Framework/Utils/PluginPathUtils.cs

@@ -0,0 +1,41 @@
+namespace Utils
+{
+    public static class PluginPathUtils
+    {
+        private const string ProjectPluginsFolderName = "ProjectPlugins";
+        private static string projectPluginsDir = string.Empty;
+
+        public static string ProjectPluginsDir
+        {
+            get
+            {
+                if (string.IsNullOrEmpty(projectPluginsDir)) projectPluginsDir = FindProjectPluginsDir();
+                return projectPluginsDir;
+            }
+        }
+
+        private static string FindProjectPluginsDir()
+        {
+            var current = Directory.GetCurrentDirectory();
+            while (true)
+            {
+                var localFolders = Directory.GetDirectories(current);
+                var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
+                if (projectPluginsFolders.Length == 1)
+                {
+                    return projectPluginsFolders.Single();
+                }
+
+                var parent = Directory.GetParent(current);
+                if (parent == null)
+                {
+                    var msg = $"Unable to locate '{ProjectPluginsFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
+                    Console.WriteLine(msg);
+                    throw new Exception(msg);
+                }
+
+                current = parent.FullName;
+            }
+        }
+    }
+}

Framework/Utils/RandomUtils.cs

@@ -3,24 +3,41 @@
     public static class RandomUtils
     {
         private static readonly Random random = new Random();
+        private static readonly object @lock = new object();
+
+        public static T GetOneRandom<T>(this T[] items)
+        {
+            lock (@lock)
+            {
+                var i = random.Next(0, items.Length);
+                var result = items[i];
+                return result;
+            }
+        }
 
         public static T PickOneRandom<T>(this List<T> remainingItems)
         {
-            var i = random.Next(0, remainingItems.Count);
-            var result = remainingItems[i];
-            remainingItems.RemoveAt(i);
-            return result;
+            lock (@lock)
+            {
+                var i = random.Next(0, remainingItems.Count);
+                var result = remainingItems[i];
+                remainingItems.RemoveAt(i);
+                return result;
+            }
         }
 
         public static T[] Shuffled<T>(T[] items)
         {
-            var result = new List<T>();
-            var source = items.ToList();
-            while (source.Any())
+            lock (@lock)
             {
-                result.Add(RandomUtils.PickOneRandom(source));
+                var result = new List<T>();
+                var source = items.ToList();
+                while (source.Any())
+                {
+                    result.Add(RandomUtils.PickOneRandom(source));
+                }
+                return result.ToArray();
             }
-            return result.ToArray();
         }
     }
 }

Framework/Utils/Retry.cs

@@ -71,6 +71,10 @@
                         task();
                         return;
                     }
+                    catch (OperationCanceledException)
+                    {
+                        return;
+                    }
                     catch (Exception ex)
                     {
                         var failure = CaptureFailure(ex);
@@ -94,7 +98,7 @@
 
             private void Fail()
             {
-                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport}",
+                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport()}",
                         new AggregateException(failures.Select(f => f.Exception)));
             }
 

Framework/Utils/Time.cs

@@ -33,6 +33,11 @@
             result += $"{d.Seconds} secs";
             return result;
         }
+
+        public static string FormatTimestamp(DateTime d)
+        {
+            return d.ToString("o");
+        }
         
         public static TimeSpan ParseTimespan(string span)
         {

Framework/Utils/Utils.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Utils</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin.Marketplace;
+using System.Collections.Generic;
 using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
@@ -12,7 +13,8 @@ namespace CodexContractsPlugin.ChainMonitor
             RequestCancelledEventDTO[] cancelled,
             RequestFailedEventDTO[] failed,
             SlotFilledEventDTO[] slotFilled,
-            SlotFreedEventDTO[] slotFreed
+            SlotFreedEventDTO[] slotFreed,
+            SlotReservationsFullEventDTO[] slotReservationsFull
             )
         {
             BlockInterval = blockInterval;
@@ -22,6 +24,9 @@ namespace CodexContractsPlugin.ChainMonitor
             Failed = failed;
             SlotFilled = slotFilled;
             SlotFreed = slotFreed;
+            SlotReservationsFull = slotReservationsFull;
+
+            All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull);
         }
 
         public BlockInterval BlockInterval { get; }
@@ -31,21 +36,8 @@ namespace CodexContractsPlugin.ChainMonitor
         public RequestFailedEventDTO[] Failed { get; }
         public SlotFilledEventDTO[] SlotFilled { get; }
         public SlotFreedEventDTO[] SlotFreed { get; }
-
-        public IHasBlock[] All
-        {
-            get
-            {
-                var all = new List<IHasBlock>();
-                all.AddRange(Requests);
-                all.AddRange(Fulfilled);
-                all.AddRange(Cancelled);
-                all.AddRange(Failed);
-                all.AddRange(SlotFilled);
-                all.AddRange(SlotFreed);
-                return all.ToArray();
-            }
-        }
+        public SlotReservationsFullEventDTO[] SlotReservationsFull { get; }
+        public IHasBlock[] All { get; }
 
         public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
         {
@@ -66,8 +58,19 @@ namespace CodexContractsPlugin.ChainMonitor
                 events.GetRequestCancelledEvents(),
                 events.GetRequestFailedEvents(),
                 events.GetSlotFilledEvents(),
-                events.GetSlotFreedEvents()
+                events.GetSlotFreedEvents(),
+                events.GetSlotReservationsFull()
             );
         }
+
+        private T[] ConcatAll<T>(params T[][] arrays)
+        {
+            var result = Array.Empty<T>();
+            foreach (var array in arrays)
+            {
+                result = result.Concat(array).ToArray();
+            }
+            return result;
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -1,7 +1,7 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
-using NethereumWorkflow.BlockUtils;
 using System.Numerics;
 using Utils;
 
@@ -16,6 +16,9 @@ namespace CodexContractsPlugin.ChainMonitor
         void OnRequestFailed(RequestEvent requestEvent);
         void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
         void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
+        void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
+
+        void OnError(string msg);
     }
 
     public class RequestEvent
@@ -48,30 +51,36 @@ namespace CodexContractsPlugin.ChainMonitor
         public TimeRange TotalSpan { get; private set; }
         public IChainStateRequest[] Requests => requests.ToArray();
 
-        public void Update()
+        public int Update()
         {
-            Update(DateTime.UtcNow);
+            return Update(DateTime.UtcNow);
         }
 
-        public void Update(DateTime toUtc)
+        public int Update(DateTime toUtc)
         {
             var span = new TimeRange(TotalSpan.To, toUtc);
             var events = ChainEvents.FromTimeRange(contracts, span);
             Apply(events);
 
             TotalSpan = new TimeRange(TotalSpan.From, span.To);
+            return events.All.Length;
         }
 
         private void Apply(ChainEvents events)
         {
             if (events.BlockInterval.TimeRange.From < TotalSpan.From)
-                throw new Exception("Attempt to update ChainState with set of events from before its current record.");
+            {
+                var msg = "Attempt to update ChainState with set of events from before its current record.";
+                handler.OnError(msg);
+                throw new Exception(msg);
+            }
 
-            log.Log($"ChainState updating: {events.BlockInterval}");
+            log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
 
             // Run through each block and apply the events to the state in order.
             var span = events.BlockInterval.TimeRange.Duration;
             var numBlocks = events.BlockInterval.NumberOfBlocks;
+            if (numBlocks == 0) return;
             var spanPerBlock = span / numBlocks;
 
             var eventUtc = events.BlockInterval.TimeRange.From;
@@ -108,7 +117,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFulfilledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
             handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
@@ -116,7 +125,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestCancelledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
             handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
@@ -124,7 +133,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFailedEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Failed);
             handler.OnRequestFailed(new RequestEvent(@event.Block, r));
@@ -132,7 +141,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFilledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
@@ -141,13 +150,21 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFreedEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.Hosts.RemoveHost((int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
             handler.OnSlotFreed(new RequestEvent(@event.Block, r), @event.SlotIndex);
         }
 
+        private void ApplyEvent(SlotReservationsFullEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.Log($"[{@event.Block.BlockNumber}] SlotReservationsFull (slotIndex:{@event.SlotIndex})");
+            handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
+        }
+
         private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
         {
             foreach (var r in requests)
@@ -161,10 +178,23 @@ namespace CodexContractsPlugin.ChainMonitor
             }
         }
 
-        private ChainStateRequest? FindRequest(byte[] requestId)
+        private ChainStateRequest? FindRequest(IHasRequestId request)
         {
-            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
-            if (r == null) log.Log("Unable to find request by ID!");
+            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
+            if (r == null)
+            {
+                var blockNumber = "unknown";
+                if (request is IHasBlock blk)
+                {
+                    blockNumber = blk.Block.BlockNumber.ToString();
+                }
+
+                var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
+                    $"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
+
+                log.Error(msg);
+                handler.OnError(msg);
+            }
             return r;
         }
 

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -1,10 +1,5 @@
 using GethPlugin;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Numerics;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
@@ -51,5 +46,15 @@ namespace CodexContractsPlugin.ChainMonitor
         {
             foreach (var handler in Handlers) handler.OnSlotFreed(requestEvent, slotIndex);
         }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotReservationsFull(requestEvent, slotIndex);
+        }
+
+        public void OnError(string msg)
+        {
+            foreach (var handler in Handlers) handler.OnError(msg);
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -32,5 +32,13 @@ namespace CodexContractsPlugin.ChainMonitor
         public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
         }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public void OnError(string msg)
+        {
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -1,7 +1,9 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.ABI;
+using Nethereum.Hex.HexConvertors.Extensions;
 using Nethereum.Util;
 using NethereumWorkflow;
 using Newtonsoft.Json;
@@ -24,6 +26,7 @@ namespace CodexContractsPlugin
         ICodexContractsEvents GetEvents(BlockInterval blockInterval);
         EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
         RequestState GetRequestState(Request request);
+        void WaitUntilNextPeriod();
     }
 
     [JsonConverter(typeof(StringEnumConverter))]
@@ -114,6 +117,15 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
+        public void WaitUntilNextPeriod()
+        {
+            log.Log("Waiting until next proof period...");
+            var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
+            var periodSeconds = (int)Deployment.Config.Proofs.Period;
+            var secondsLeft = now % periodSeconds;
+            Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1));
+        }
+
         private ContractInteractions StartInteraction()
         {
             return new ContractInteractions(log, gethNode);

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -19,7 +19,7 @@ namespace CodexContractsPlugin
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 
-            var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
+            var address = config.GethNode.StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
 
             SetSchedulingAffinity(notIn: "false");
 

ProjectPlugins/CodexContractsPlugin/CodexContractsDeployment.cs

@@ -1,14 +1,18 @@
-namespace CodexContractsPlugin
+using CodexContractsPlugin.Marketplace;
+
+namespace CodexContractsPlugin
 {
     public class CodexContractsDeployment
     {
-        public CodexContractsDeployment(string marketplaceAddress, string abi, string tokenAddress)
+        public CodexContractsDeployment(MarketplaceConfig config, string marketplaceAddress, string abi, string tokenAddress)
         {
+            Config = config;
             MarketplaceAddress = marketplaceAddress;
             Abi = abi;
             TokenAddress = tokenAddress;
         }
 
+        public MarketplaceConfig Config { get; }
         public string MarketplaceAddress { get; }
         public string Abi { get; }
         public string TokenAddress { get; }

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -1,8 +1,9 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
+using Nethereum.Contracts;
 using Nethereum.Hex.HexTypes;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -16,6 +17,7 @@ namespace CodexContractsPlugin
         RequestFailedEventDTO[] GetRequestFailedEvents();
         SlotFilledEventDTO[] GetSlotFilledEvents();
         SlotFreedEventDTO[] GetSlotFreedEvents();
+        SlotReservationsFullEventDTO[] GetSlotReservationsFull();
     }
 
     public class CodexContractsEvents : ICodexContractsEvents
@@ -38,49 +40,32 @@ namespace CodexContractsPlugin
         {
             var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
             var i = new ContractInteractions(log, gethNode);
-            return events
-            .Select(e =>
+            return events.Select(e =>
             {
-                        var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
-                        var result = requestEvent.ReturnValue1;
-                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                        result.RequestId = e.Event.RequestId;
-                        return result;
-                    })
-                    .ToArray();
+                var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
+                var result = requestEvent.ReturnValue1;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.RequestId = e.Event.RequestId;
+                return result;
+            }).ToArray();
         }
 
         public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()
         {
             var events = gethNode.GetEvents<RequestFulfilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public RequestCancelledEventDTO[] GetRequestCancelledEvents()
         {
             var events = gethNode.GetEvents<RequestCancelledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public RequestFailedEventDTO[] GetRequestFailedEvents()
         {
             var events = gethNode.GetEvents<RequestFailedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public SlotFilledEventDTO[] GetSlotFilledEvents()
@@ -98,12 +83,20 @@ namespace CodexContractsPlugin
         public SlotFreedEventDTO[] GetSlotFreedEvents()
         {
             var events = gethNode.GetEvents<SlotFreedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        public SlotReservationsFullEventDTO[] GetSlotReservationsFull()
+        {
+            var events = gethNode.GetEvents<SlotReservationsFullEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
+        {
+            var result = e.Event;
+            result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+            return result;
         }
 
         private BlockTimeEntry GetBlock(ulong number)

ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -4,6 +4,7 @@ using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
+using Newtonsoft.Json;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -34,6 +35,7 @@ namespace CodexContractsPlugin
             try
             {
                 var result = DeployContract(container, workflow, gethNode);
+
                 workflow.Stop(containers, waitTillStopped: false);
                 Log("Container stopped.");
                 return result;
@@ -75,9 +77,20 @@ namespace CodexContractsPlugin
 
             Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract));
 
-            Log("Synced. Codex SmartContracts deployed.");
+            Log("Synced. Codex SmartContracts deployed. Getting configuration...");
 
-            return new CodexContractsDeployment(marketplaceAddress, abi, tokenAddress);
+            var config = GetMarketplaceConfiguration(marketplaceAddress, gethNode);
+
+            Log("Got config: " + JsonConvert.SerializeObject(config));
+
+            return new CodexContractsDeployment(config, marketplaceAddress, abi, tokenAddress);
+        }
+
+        private MarketplaceConfig GetMarketplaceConfiguration(string marketplaceAddress, IGethNode gethNode)
+        {
+            var func = new ConfigurationFunctionBase();
+            var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
+            return response.ReturnValue1;
         }
 
         private void EnsureCompatbility(string abi, string bytecode)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
@@ -23,9 +24,8 @@ namespace CodexContractsPlugin
         public string GetTokenAddress(string marketplaceAddress)
         {
             log.Debug(marketplaceAddress);
-            var function = new GetTokenFunction();
-
-            return gethNode.Call<GetTokenFunction, string>(marketplaceAddress, function);
+            var function = new TokenFunctionBase();
+            return gethNode.Call<TokenFunctionBase, string>(marketplaceAddress, function);
         }
 
         public string GetTokenName(string tokenAddress)
@@ -111,11 +111,6 @@ namespace CodexContractsPlugin
         }
     }
 
-    [Function("token", "address")]
-    public class GetTokenFunction : FunctionMessage
-    {
-    }
-
     [Function("name", "string")]
     public class GetTokenNameFunction : FunctionMessage
     {

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,6 +1,6 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
+using BlockchainUtils;
 using GethPlugin;
-using NethereumWorkflow.BlockUtils;
 using Newtonsoft.Json;
 
 namespace CodexContractsPlugin.Marketplace
@@ -10,7 +10,12 @@ namespace CodexContractsPlugin.Marketplace
         BlockTimeEntry Block { get; set; }
     }
 
-    public partial class Request : RequestBase, IHasBlock
+    public interface IHasRequestId
+    {
+        byte[] RequestId { get; set; }
+    }
+
+    public partial class Request : RequestBase, IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
@@ -28,32 +33,38 @@ namespace CodexContractsPlugin.Marketplace
         }
     }
 
-    public partial class RequestFulfilledEventDTO : IHasBlock
+    public partial class RequestFulfilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestCancelledEventDTO : IHasBlock
+    public partial class RequestCancelledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestFailedEventDTO : IHasBlock
+    public partial class RequestFailedEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class SlotFilledEventDTO : IHasBlock
+    public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
     }
 
-    public partial class SlotFreedEventDTO : IHasBlock
+    public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
+
+    public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }

ProjectPlugins/CodexContractsPlugin/SelfUpdater.cs

@@ -1,4 +1,6 @@
-namespace CodexContractsPlugin
+using Utils;
+
+namespace CodexContractsPlugin
 {
     public class SelfUpdater
     {
@@ -41,24 +43,10 @@
 
         private string GetMarketplaceFilePath()
         {
-            var here = Directory.GetCurrentDirectory();
-            while (true)
-            {
-                var path = GetMarketplaceFile(here);
-                if (path != null) return path;
-
-                var parent = Directory.GetParent(here);
-                var up = parent?.FullName;
-                if (up == null || up == here) throw new Exception("Unable to locate ProjectPlugins folder. Unable to update contracts.");
-                here = up;
-            }
-        }
-
-        private string? GetMarketplaceFile(string root)
-        {
-            var path = Path.Combine(root, "ProjectPlugins", "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
-            if (File.Exists(path)) return path;
-            return null;
+            var projectPluginDir = PluginPathUtils.ProjectPluginsDir;
+            var path = Path.Combine(projectPluginDir, "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
+            if (!File.Exists(path)) throw new Exception("Marketplace file not found. Expected: " + path);
+            return path;
         }
 
         private string GenerateContent(string abi, string bytecode)

ProjectPlugins/CodexContractsPlugin/TestTokenExtensions.cs

@@ -46,6 +46,16 @@ namespace CodexContractsPlugin
             return new TestToken(a.TstWei + b.TstWei);
         }
 
+        public static TestToken operator -(TestToken a, TestToken b)
+        {
+            return new TestToken(a.TstWei - b.TstWei);
+        }
+
+        public static TestToken operator *(TestToken a, int b)
+        {
+            return new TestToken(a.TstWei * b);
+        }
+
         public static bool operator <(TestToken a, TestToken b)
         {
             return a.TstWei < b.TstWei;

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -3,13 +3,14 @@ using KubernetesWorkflow.Types;
 using Logging;
 using System.Security.Cryptography;
 using System.Text;
+using Utils;
 
 namespace CodexPlugin
 {
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "67-76-AB-FC-54-4F-EB-81-F5-E4-F8-27-DF-82-92-41-63-A5-EA-1B-17-14-0C-BE-20-9C-B3-DF-CE-E4-AA-38";
+        private const string OpenApiYamlHash = "34-B5-DA-26-40-76-B8-D8-8E-7D-9C-17-85-C6-B0-63-55-8D-C6-01-0B-96-BB-7C-BD-53-E5-32-07-ED-29-92";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 
@@ -21,8 +22,9 @@ namespace CodexPlugin
 
         private const string Failure =
             "Codex API compatibility check failed! " +
-            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. Please update the openapi.yaml in " +
-            "'ProjectPlugins/CodexPlugin' and rebuild this project. If you wish to disable API compatibility checking, please set " +
+            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. The openapi.yaml in " +
+            "'ProjectPlugins/CodexPlugin' has been overwritten with the container one. " +
+            "Please and rebuild this project. If you wish to disable API compatibility checking, please set " +
             $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/CodexPlugin/ApiChecker.cs'.";
 
         private static bool checkPassed = false;
@@ -71,10 +73,23 @@ namespace CodexPlugin
                 return;
             }
 
+            OverwriteOpenApiYaml(containerApi);
+
             log.Error(Failure);
             throw new Exception(Failure);
         }
 
+        private void OverwriteOpenApiYaml(string containerApi)
+        {
+            Log("API compatibility check failed. Updating CodexPlugin...");
+            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin", "openapi.yaml");
+            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexPlugin/openapi.yaml. Expected: " + openApiFilePath);
+
+            File.Delete(openApiFilePath);
+            File.WriteAllText(openApiFilePath, containerApi);
+            Log("CodexPlugin/openapi.yaml has been updated.");
+        }
+
         private string Hash(string file)
         {
             var fileBytes = Encoding.ASCII.GetBytes(file

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -32,6 +32,22 @@ namespace CodexPlugin
             return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
         }
 
+        public string GetSpr()
+        {
+            return CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                var json = endpoint.HttpGetString("spr");
+                var response = JsonConvert.DeserializeObject<SprResponse>(json);
+                return response!.Spr;
+            });
+        }
+
+        private class SprResponse
+        {
+            public string Spr { get; set; } = string.Empty;
+        }
+
         public DebugPeer GetDebugPeer(string peerId)
         {
             // Cannot use openAPI: debug/peer endpoint is not specified there.
@@ -63,44 +79,75 @@ namespace CodexPlugin
             });
         }
 
-        public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
+        public string UploadFile(UploadInput uploadInput, Action<Failure> onFailure)
         {
             return OnCodex(
-                api => api.UploadAsync(fileStream),
+                api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream),
                 CreateRetryConfig(nameof(UploadFile), onFailure));
         }
 
         public Stream DownloadFile(string contentId, Action<Failure> onFailure)
         {
             var fileResponse = OnCodex(
-                api => api.DownloadNetworkAsync(contentId),
+                api => api.DownloadNetworkStreamAsync(contentId),
                 CreateRetryConfig(nameof(DownloadFile), onFailure));
 
             if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
             return fileResponse.Stream;
         }
 
+        public LocalDataset DownloadStreamless(ContentId cid)
+        {
+            var response = OnCodex(api => api.DownloadNetworkAsync(cid.Id));
+            return mapper.Map(response);
+        }
+
+        public LocalDataset DownloadManifestOnly(ContentId cid)
+        {
+            var response = OnCodex(api => api.DownloadNetworkManifestAsync(cid.Id));
+            return mapper.Map(response);
+        }
+
         public LocalDatasetList LocalFiles()
         {
-            return mapper.Map(OnCodex(api => api.ListDataAsync()));
+            // API for listData mismatches.
+            //return mapper.Map(OnCodex(api => api.ListDataAsync()));
+
+            return mapper.Map(CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                return Time.Retry(() =>
+                {
+                    var str = endpoint.HttpGetString("data");
+                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
+                    return JsonConvert.DeserializeObject<LocalDatasetListJson>(str)!;
+                }, nameof(LocalFiles));
+            }));
+
         }
 
         public StorageAvailability SalesAvailability(StorageAvailability request)
         {
             var body = mapper.Map(request);
-            var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
+            var read = OnCodex(api => api.OfferStorageAsync(body));
             return mapper.Map(read);
         }
 
+        public StorageAvailability[] GetAvailabilities()
+        {
+            var collection = OnCodex(api => api.GetAvailabilitiesAsync());
+            return mapper.Map(collection);
+        }
+
         public string RequestStorage(StoragePurchaseRequest request)
         {
             var body = mapper.Map(request);
-            return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
+            return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
         public CodexSpace Space()
         {
-            var space = OnCodex<Space>(api => api.SpaceAsync());
+            var space = OnCodex(api => api.SpaceAsync());
             return mapper.Map(space);
         }
 
@@ -187,9 +234,16 @@ namespace CodexPlugin
                 .CreateEndpoint(GetAddress(), "/api/codex/v1/", Container.Name);
         }
 
+        public static Address? UploaderOverride { get; set; } = null;
+        public static Address? DownloaderOverride { get; set; } = null;
+
         private Address GetAddress()
         {
-            return Container.Containers.Single().GetAddress(log, CodexContainerRecipe.ApiPortTag);
+            if (GetName().ToLowerInvariant().Contains("upload") && UploaderOverride != null) return UploaderOverride;
+            if (GetName().ToLowerInvariant().Contains("download") && DownloaderOverride != null) return DownloaderOverride;
+
+
+            return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
         }
 
         private string GetHttpId()
@@ -255,4 +309,18 @@ namespace CodexPlugin
             log.Log($"{GetName()} {msg}");
         }
     }
+
+    public class UploadInput
+    {
+        public UploadInput(string contentType, string contentDisposition, FileStream fileStream)
+        {
+            ContentType = contentType;
+            ContentDisposition = contentDisposition;
+            FileStream = fileStream;
+        }
+
+        public string ContentType { get; }
+        public string ContentDisposition { get; }
+        public FileStream FileStream { get; }
+    }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,8 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-1e2ad95-dist-tests";
-
+        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";
@@ -110,7 +109,7 @@ namespace CodexPlugin
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
                 var account = marketplaceSetup.EthAccountSetup.GetNew();
-                AddEnvVar("PRIV_KEY", account.PrivateKey);
+                AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
                 Additional(account);
 
                 SetCommandOverride(marketplaceSetup);

ProjectPlugins/CodexPlugin/CodexLogLine.cs

@@ -48,6 +48,11 @@ namespace CodexPlugin
         public string Message { get; set; } = string.Empty;
         public Dictionary<string, string> Attributes { get; private set; } = new Dictionary<string, string>();
 
+        public override string ToString()
+        {
+            return Message;
+        }
+
         /// <summary>
         /// After too much time spent cursing at regexes, here's what I got:
         /// Parses input string into 'key=value' pair, considerate of quoted (") values.

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -14,12 +14,23 @@ namespace CodexPlugin
     {
         string GetName();
         string GetPeerId();
-        DebugInfo GetDebugInfo();
+        DebugInfo GetDebugInfo(bool log = false);
+        string GetSpr();
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
         ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
+        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
+        (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, string fileLabel = "");
+        (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
+        LocalDataset DownloadStreamless(ContentId cid);
+        /// <summary>
+        /// TODO: This will monitor the quota-used of the node until 'size' bytes are added. That's a very bad way
+        /// to track the streamless download progress. Replace it once we have a good API for this.
+        /// </summary>
+        LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size);
+        LocalDataset DownloadManifestOnly(ContentId cid);
         LocalDatasetList LocalFiles();
         CodexSpace Space();
         void ConnectToPeer(ICodexNode node);
@@ -119,14 +130,22 @@ namespace CodexPlugin
             return peerId;
         }
 
-        public DebugInfo GetDebugInfo()
+        public DebugInfo GetDebugInfo(bool log = false)
         {
             var debugInfo = CodexAccess.GetDebugInfo();
-            var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
-            Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
+            if (log)
+            {
+                var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
+                Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
+            }
             return debugInfo;
         }
 
+        public string GetSpr()
+        {
+            return CodexAccess.GetSpr();
+        }
+
         public DebugPeer GetDebugPeer(string peerId)
         {
             return CodexAccess.GetDebugPeer(peerId);
@@ -138,6 +157,11 @@ namespace CodexPlugin
         }
 
         public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
+        {
+            return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"", onFailure);
+        }
+
+        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenRead(file.Filename);
             var uniqueId = Guid.NewGuid().ToString();
@@ -145,10 +169,11 @@ namespace CodexPlugin
 
             hooks.OnFileUploading(uniqueId, size);
 
-            var logMessage = $"Uploading file {file.Describe()}...";
+            var input = new UploadInput(contentType, contentDisposition, fileStream);
+            var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
             var measurement = Stopwatch.Measure(log, logMessage, () =>
             {
-                return CodexAccess.UploadFile(fileStream, onFailure);
+                return CodexAccess.UploadFile(input, onFailure);
             });
 
             var response = measurement.Value;
@@ -166,10 +191,20 @@ namespace CodexPlugin
 
         public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
         {
-            return DownloadContent(contentId, DoNothing, fileLabel);
+            return DownloadContentT(contentId, fileLabel).Item1;
         }
 
         public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
+        {
+            return DownloadContentT(contentId, onFailure, fileLabel).Item1;
+        }
+
+        public (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, string fileLabel = "")
+        {
+            return DownloadContentT(contentId, DoNothing, fileLabel);
+        }
+
+        public (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
         {
             var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
             hooks.OnFileDownloading(contentId);
@@ -182,7 +217,34 @@ namespace CodexPlugin
             transferSpeeds.AddDownloadSample(size, measurement);
             hooks.OnFileDownloaded(size, contentId);
 
-            return file;
+            return (file, measurement);
+        }
+
+        public LocalDataset DownloadStreamless(ContentId cid)
+        {
+            Log($"Downloading streamless '{cid}' (no-wait)");
+            return CodexAccess.DownloadStreamless(cid);
+        }
+
+        public LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size)
+        {
+            Log($"Downloading streamless '{cid}' (wait till finished)");
+
+            var sw = Stopwatch.Measure(log, nameof(DownloadStreamlessWait), () =>
+            {
+                var startSpace = Space();
+                var result = CodexAccess.DownloadStreamless(cid);
+                WaitUntilQuotaUsedIncreased(startSpace, size);
+                return result;
+            });
+
+            return sw.Value;
+        }
+
+        public LocalDataset DownloadManifestOnly(ContentId cid)
+        {
+            Log($"Downloading manifest-only '{cid}'");
+            return CodexAccess.DownloadManifestOnly(cid);
         }
 
         public LocalDatasetList LocalFiles()
@@ -244,6 +306,11 @@ namespace CodexPlugin
             Version = debugInfo.Version;
         }
 
+        public override string ToString()
+        {
+            return $"CodexNode:{GetName()}";
+        }
+
         private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)
         {
             // The peer we want to connect is in a different pod.
@@ -259,10 +326,27 @@ namespace CodexPlugin
         private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenWrite(file.Filename);
+            var timeout = tools.TimeSet.HttpCallTimeout();
             try
             {
-                using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
-                downloadStream.CopyTo(fileStream);
+                // Type of stream generated by openAPI client does not support timeouts.
+                var start = DateTime.UtcNow;
+                var cts = new CancellationTokenSource();
+                var downloadTask = Task.Run(() =>
+                {
+                    using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
+                    downloadStream.CopyTo(fileStream);
+                }, cts.Token);
+                
+                while (DateTime.UtcNow - start < timeout)
+                {
+                    if (downloadTask.IsFaulted) throw downloadTask.Exception;
+                    if (downloadTask.IsCompletedSuccessfully) return;
+                    Thread.Sleep(100);
+                }
+
+                cts.Cancel();
+                throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
             }
             catch
             {
@@ -271,6 +355,39 @@ namespace CodexPlugin
             }
         }
 
+        public void WaitUntilQuotaUsedIncreased(CodexSpace startSpace, ByteSize expectedIncreaseOfQuotaUsed)
+        {
+            WaitUntilQuotaUsedIncreased(startSpace, expectedIncreaseOfQuotaUsed, TimeSpan.FromMinutes(2));
+        }
+
+        public void WaitUntilQuotaUsedIncreased(
+            CodexSpace startSpace,
+            ByteSize expectedIncreaseOfQuotaUsed,
+            TimeSpan maxTimeout)
+        {
+            Log($"Waiting until quotaUsed " +
+                $"(start: {startSpace.QuotaUsedBytes}) " +
+                $"increases by {expectedIncreaseOfQuotaUsed} " +
+                $"to reach {startSpace.QuotaUsedBytes + expectedIncreaseOfQuotaUsed.SizeInBytes}");
+
+            var retry = new Retry($"Checking local space for quotaUsed increase of {expectedIncreaseOfQuotaUsed}",
+            maxTimeout: maxTimeout,
+            sleepAfterFail: TimeSpan.FromSeconds(3),
+            onFail: f => { });
+
+            retry.Run(() =>
+            {
+                var space = Space();
+                var increase = space.QuotaUsedBytes - startSpace.QuotaUsedBytes;
+
+                if (increase < expectedIncreaseOfQuotaUsed.SizeInBytes)
+                    throw new Exception($"Expected quota-used not reached. " +
+                        $"Expected increase: {expectedIncreaseOfQuotaUsed.SizeInBytes} " +
+                        $"Actual increase: {increase} " +
+                        $"Actual used: {space.QuotaUsedBytes}");
+            });
+        }
+
         private void EnsureMarketplace()
         {
             if (ethAccount == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -20,7 +20,7 @@ namespace CodexPlugin
 
         public void Announce()
         {
-            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
+            Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
         }
 
         public void AddMetadata(IAddMetadata metadata)
@@ -55,6 +55,10 @@ namespace CodexPlugin
             {
                 mconfig.GethNode.SendEth(node, mconfig.MarketplaceSetup.InitialEth);
                 mconfig.CodexContracts.MintTestTokens(node, mconfig.MarketplaceSetup.InitialTestTokens);
+
+                Log($"Send {mconfig.MarketplaceSetup.InitialEth} and " +
+                    $"minted {mconfig.MarketplaceSetup.InitialTestTokens} for " +
+                    $"{node.GetName()} (address: {node.EthAddress})");
             }
         }
 
@@ -70,5 +74,10 @@ namespace CodexPlugin
             setup(codexSetup);
             return codexSetup;
         }
+
+        private void Log(string msg)
+        {
+            tools.GetLog().Log(msg);
+        }
     }
 }

ProjectPlugins/CodexPlugin/CodexPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -16,8 +16,16 @@ namespace CodexPlugin
                 Spr = debugInfo.Spr,
                 Addrs = debugInfo.Addrs.ToArray(),
                 AnnounceAddresses = JArray(debugInfo.AdditionalProperties, "announceAddresses").Select(x => x.ToString()).ToArray(),
-                Version = MapDebugInfoVersion(JObject(debugInfo.AdditionalProperties, "codex")),
-                Table = MapDebugInfoTable(JObject(debugInfo.AdditionalProperties, "table"))
+                Version = Map(debugInfo.Codex),
+                Table = Map(debugInfo.Table)
+            };
+        }
+
+        public LocalDatasetList Map(LocalDatasetListJson json)
+        {
+            return new LocalDatasetList
+            {
+                Content = json.Content.Select(Map).ToArray()
             };
         }
 
@@ -38,6 +46,15 @@ namespace CodexPlugin
             };
         }
 
+        public LocalDataset Map(LocalDatasetListJsonItem item)
+        {
+            return new LocalDataset
+            {
+                Cid = new ContentId(item.Cid),
+                Manifest = MapManifest(item.Manifest)
+            };
+        }
+
         public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
         {
             return new CodexOpenApi.SalesAvailabilityCREATE
@@ -63,6 +80,26 @@ namespace CodexPlugin
             };
         }
 
+        public StorageAvailability[] Map(ICollection<SalesAvailabilityREAD> availabilities)
+        {
+            return availabilities.Select(a => Map(a)).ToArray();
+        }
+
+        public StorageAvailability Map(SalesAvailabilityREAD availability)
+        {
+            return new StorageAvailability
+            (
+                ToByteSize(availability.TotalSize),
+                ToTimespan(availability.Duration),
+                new TestToken(ToBigIng(availability.MinPrice)),
+                new TestToken(ToBigIng(availability.MaxCollateral))
+            )
+            {
+                Id = availability.Id,
+                FreeSpace = ToByteSize(availability.FreeSize),
+            };
+        }
+
         // TODO: Fix openapi spec for this call.
         //public StoragePurchase Map(CodexOpenApi.Purchase purchase)
         //{
@@ -105,19 +142,6 @@ namespace CodexPlugin
         //    };
         //}
 
-        public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD read)
-        {
-            return new StorageAvailability(
-                totalSpace: new ByteSize(Convert.ToInt64(read.TotalSize)),
-                maxDuration: TimeSpan.FromSeconds(Convert.ToDouble(read.Duration)),
-                minPriceForTotalSpace: new TestToken(BigInteger.Parse(read.MinPrice)),
-                maxCollateral: new TestToken(BigInteger.Parse(read.MaxCollateral))
-            )
-            {
-                Id = read.Id
-            };
-        }
-
         public CodexSpace Map(Space space)
         {
             return new CodexSpace
@@ -129,47 +153,45 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoVersion MapDebugInfoVersion(JObject obj)
+        private DebugInfoVersion Map(CodexVersion obj)
         {
             return new DebugInfoVersion
             {
-                Version = StringOrEmpty(obj, "version"),
-                Revision = StringOrEmpty(obj, "revision")
+                Version = obj.Version,
+                Revision = obj.Revision
             };
         }
 
-        private DebugInfoTable MapDebugInfoTable(JObject obj)
+        private DebugInfoTable Map(PeersTable obj)
         {
             return new DebugInfoTable
             {
-                LocalNode = MapDebugInfoTableNode(obj.GetValue("localNode")),
-                Nodes = MapDebugInfoTableNodeArray(obj.GetValue("nodes") as JArray)
+                LocalNode = Map(obj.LocalNode),
+                Nodes = Map(obj.Nodes)
             };
         }
 
-        private DebugInfoTableNode MapDebugInfoTableNode(JToken? token)
+        private DebugInfoTableNode Map(Node? token)
         {
-            var obj = token as JObject;
-            if (obj == null) return new DebugInfoTableNode();
-
+            if (token == null) return new DebugInfoTableNode();
             return new DebugInfoTableNode
             {
-                Address = StringOrEmpty(obj, "address"),
-                NodeId = StringOrEmpty(obj, "nodeId"),
-                PeerId = StringOrEmpty(obj, "peerId"),
-                Record = StringOrEmpty(obj, "record"),
-                Seen = Bool(obj, "seen")
+                Address = token.Address,
+                NodeId = token.NodeId,
+                PeerId = token.PeerId,
+                Record = token.Record,
+                Seen = token.Seen
             };
         }
 
-        private DebugInfoTableNode[] MapDebugInfoTableNodeArray(JArray? nodes)
+        private DebugInfoTableNode[] Map(ICollection<Node> nodes)
         {
             if (nodes == null || nodes.Count == 0)
             {
                 return new DebugInfoTableNode[0];
             }
 
-            return nodes.Select(MapDebugInfoTableNode).ToArray();
+            return nodes.Select(Map).ToArray();
         }
 
         private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)
@@ -177,8 +199,20 @@ namespace CodexPlugin
             return new Manifest
             {
                 BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
-                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.OriginalBytes)),
-                RootHash = manifest.RootHash,
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
+                RootHash = manifest.TreeCid,
+                Protected = manifest.Protected
+            };
+        }
+
+        public Manifest MapManifest(LocalDatasetListJsonItemManifest manifest)
+        {
+            return new Manifest
+            {
+                // needs update
+                BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
+                RootHash = manifest.TreeCid,
                 Protected = manifest.Protected
             };
         }
@@ -222,5 +256,58 @@ namespace CodexPlugin
         {
             return t.TstWei.ToString("D");
         }
+
+        private BigInteger ToBigIng(string tokens)
+        {
+            return BigInteger.Parse(tokens);
+        }
+
+        private TimeSpan ToTimespan(string duration)
+        {
+            return TimeSpan.FromSeconds(Convert.ToInt32(duration));
+        }
+
+        private ByteSize ToByteSize(string size)
+        {
+            return new ByteSize(Convert.ToInt64(size));
+        }
+    }
+
+
+//"content": [
+//        {
+//            "cid": "zDvZRwzkxLxVaGces3kpkHjo8EcTPXudvYMfNxdoH21Ask1Js5fJ",
+//            "manifest": {
+//                "treeCid": "zDzSvJTf8GBRyEDNuAzXS9VnRfh8cNuYuRPwTLW6RUQReSgKnhCt",
+//                "datasetSize": 5242880,
+//                "blockSize": 65536,
+//                "filename": null,
+//                "mimetype": "application/octet-stream",
+//                "uploadedAt": 1731426230,
+//                "protected": false
+//            }
+//        }
+//    ]
+
+    public class LocalDatasetListJson
+    {
+        public LocalDatasetListJsonItem[] Content { get; set; } = Array.Empty<LocalDatasetListJsonItem>();
+    }
+
+    public class LocalDatasetListJsonItem
+    {
+        public string Cid { get; set; } = string.Empty;
+        public LocalDatasetListJsonItemManifest Manifest { get; set; } = new();
+    }
+
+    public class LocalDatasetListJsonItemManifest
+    {
+        public string TreeCid { get; set; } = string.Empty;
+        public int DatasetSize { get; set; }
+        public int BlockSize { get; set; }
+        public string? Filename { get; set; } = string.Empty;
+        public string? MimeType { get; set; } = string.Empty;
+        public int? UploadedAt { get; set; } 
+        public bool Protected { get; set; }
     }
 }

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -7,6 +7,7 @@ namespace CodexPlugin
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(StorageAvailability availability);
+        StorageAvailability[] GetAvailabilities();
         IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
     }
 
@@ -61,6 +62,14 @@ namespace CodexPlugin
             return response.Id;
         }
 
+        public StorageAvailability[] GetAvailabilities()
+        {
+            var result = codexAccess.GetAvailabilities();
+            Log($"Got {result.Length} availabilities:");
+            foreach (var a in result) a.Log(log);
+            return result;
+        }
+
         private void Log(string msg)
         {
             log.Log($"{codexAccess.Container.Containers.Single().Name} {msg}");
@@ -81,6 +90,12 @@ namespace CodexPlugin
             throw new NotImplementedException();
         }
 
+        public StorageAvailability[] GetAvailabilities()
+        {
+            Unavailable();
+            throw new NotImplementedException();
+        }
+
         private void Unavailable()
         {
             FrameworkAssert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it.");

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -1,7 +1,5 @@
 using CodexContractsPlugin;
-using CodexOpenApi;
 using Logging;
-using System.Data;
 using Utils;
 
 namespace CodexPlugin
@@ -40,6 +38,12 @@ namespace CodexPlugin
         public string State { get; set; } = string.Empty;
         public string Error { get; set; } = string.Empty;
         public StorageRequest Request { get; set; } = null!;
+
+        public bool IsCancelled => State.ToLowerInvariant().Contains("cancel");
+        public bool IsError => State.ToLowerInvariant().Contains("error");
+        public bool IsFinished => State.ToLowerInvariant().Contains("finished");
+        public bool IsStarted => State.ToLowerInvariant().Contains("started");
+        public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
     }
 
     public class StorageRequest
@@ -84,10 +88,11 @@ namespace CodexPlugin
         public TimeSpan MaxDuration { get; }
         public TestToken MinPriceForTotalSpace { get; }
         public TestToken MaxCollateral { get; } 
+        public ByteSize FreeSpace { get; set; } = ByteSize.Zero;
 
         public void Log(ILog log)
         {
-            log.Log($"Making storage available... (" +
+            log.Log($"Storage Availability: (" +
                 $"totalSize: {TotalSpace}, " +
                 $"maxDuration: {Time.FormatDuration(MaxDuration)}, " + 
                 $"minPriceForTotalSpace: {MinPriceForTotalSpace}, " +

ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs

@@ -1,4 +1,6 @@
-using CodexPlugin.Hooks;
+using CodexContractsPlugin;
+using CodexPlugin.Hooks;
+using GethPlugin;
 using Logging;
 using Newtonsoft.Json;
 using Utils;
@@ -12,7 +14,8 @@ namespace CodexPlugin
         ContentId ContentId { get; }
         void WaitForStorageContractSubmitted();
         void WaitForStorageContractStarted();
-        void WaitForStorageContractFinished();
+        void WaitForStorageContractFinished(ICodexContracts contracts);
+        void WaitForContractFailed();
     }
 
     public class StoragePurchaseContract : IStoragePurchaseContract
@@ -62,7 +65,7 @@ namespace CodexPlugin
             AssertDuration(SubmittedToStarted, timeout, nameof(SubmittedToStarted));
         }
 
-        public void WaitForStorageContractFinished()
+        public void WaitForStorageContractFinished(ICodexContracts contracts)
         {
             if (!contractStartedUtc.HasValue)
             {
@@ -74,6 +77,24 @@ namespace CodexPlugin
             contractFinishedUtc = DateTime.UtcNow;
             LogFinishedDuration();
             AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
+
+            contracts.WaitUntilNextPeriod();
+            contracts.WaitUntilNextPeriod();
+
+            var blocks = 3;
+            Log($"Waiting {blocks} blocks for nodes to process payouts...");
+            Thread.Sleep(GethContainerRecipe.BlockInterval * blocks);
+        }
+
+        public void WaitForContractFailed()
+        {
+            if (!contractStartedUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
+            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
+            WaitForStorageContractState(timeout, "failed");
         }
 
         public StoragePurchase GetPurchaseStatus(string purchaseId)

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -23,6 +23,8 @@ components:
     Id:
       type: string
       description: 32bits identifier encoded in hex-decimal string.
+      minLength: 66
+      maxLength: 66
       example: 0x...
 
     BigInt:
@@ -81,33 +83,46 @@ components:
         id:
           $ref: "#/components/schemas/PeerId"
 
-    ErasureParameters:
-      type: object
-      properties:
-        totalChunks:
-          type: integer
-
-    PoRParameters:
-      description: Parameters for Proof of Retrievability
-      type: object
-      properties:
-        u:
-          type: string
-        publicKey:
-          type: string
-        name:
-          type: string
-
     Content:
       type: object
       description: Parameters specifying the content
       properties:
         cid:
           $ref: "#/components/schemas/Cid"
-        erasure:
-          $ref: "#/components/schemas/ErasureParameters"
-        por:
-          $ref: "#/components/schemas/PoRParameters"
+
+    Node:
+      type: object
+      properties:
+        nodeId:
+          type: string
+        peerId:
+          type: string
+        record:
+          type: string
+        address:
+          type: string
+        seen:
+          type: boolean
+
+    CodexVersion:
+      type: object
+      properties:
+        version:
+          type: string
+          example: v0.1.7
+        revision:
+          type: string
+          example: 0c647d8
+
+    PeersTable:
+      type: object
+      properties:
+        localNode:
+          $ref: "#/components/schemas/Node"
+        nodes:
+          type: array
+          items:
+            $ref: "#/components/schemas/Node"
 
     DebugInfo:
       type: object
@@ -123,6 +138,10 @@ components:
           description: Path of the data repository where all nodes data are stored
         spr:
           $ref: "#/components/schemas/SPR"
+        table:
+          $ref: "#/components/schemas/PeersTable"
+        codex:
+          $ref: "#/components/schemas/CodexVersion"
 
     SalesAvailability:
       type: object
@@ -136,7 +155,7 @@ components:
           $ref: "#/components/schemas/Duration"
         minPrice:
           type: string
-          description: Minimum price to be paid (in amount of tokens) as decimal string
+          description: Minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration as decimal string
         maxCollateral:
           type: string
           description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
@@ -168,7 +187,39 @@ components:
           $ref: "#/components/schemas/StorageRequest"
         slotIndex:
           type: string
-          description: Slot Index as hexadecimal string
+          description: Slot Index as decimal string
+
+    SlotAgent:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/SlotId"
+        slotIndex:
+          type: string
+          description: Slot Index as decimal string
+        requestId:
+          $ref: "#/components/schemas/Id"
+        request:
+          $ref: "#/components/schemas/StorageRequest"
+        reservation:
+          $ref: "#/components/schemas/Reservation"
+        state:
+          type: string
+          description: Description of the slot's
+          enum:
+            - SaleCancelled
+            - SaleDownloading
+            - SaleErrored
+            - SaleFailed
+            - SaleFilled
+            - SaleFilling
+            - SaleFinished
+            - SaleIgnored
+            - SaleInitialProving
+            - SalePayout
+            - SalePreparing
+            - SaleProving
+            - SaleUnknown
 
     Reservation:
       type: object
@@ -183,7 +234,7 @@ components:
           $ref: "#/components/schemas/Id"
         slotIndex:
           type: string
-          description: Slot Index as hexadecimal string
+          description: Slot Index as decimal string
 
     StorageRequestCreation:
       type: object
@@ -259,6 +310,15 @@ components:
         state:
           type: string
           description: Description of the Request's state
+          enum:
+            - cancelled
+            - error
+            - failed
+            - finished
+            - pending
+            - started
+            - submitted
+            - unknown
         error:
           type: string
           description: If Request failed, then here is presented the error message
@@ -284,10 +344,10 @@ components:
     ManifestItem:
       type: object
       properties:
-        rootHash:
+        treeCid:
           $ref: "#/components/schemas/Cid"
-          description: "Root hash of the content"
-        originalBytes:
+          description: "Unique data identifier"
+        datasetSize:
           type: integer
           format: int64
           description: "Length of original content in bytes"
@@ -297,6 +357,22 @@ components:
         protected:
           type: boolean
           description: "Indicates if content is protected by erasure-coding"
+        filename:
+          type: string
+          nullable: true
+          description: "The original name of the uploaded content (optional)"
+          example: codex.png
+        mimetype:
+          type: string
+          nullable: true
+          description: "The original mimetype of the uploaded content (optional)"
+          example: image/png
+        uploadedAt:
+          type: integer
+          format: int64
+          nullable: true
+          description: "The UTC upload timestamp in seconds"
+          example: 1729244192
 
     Space:
       type: object
@@ -308,15 +384,15 @@ components:
         quotaMaxBytes:
           type: integer
           format: int64
-          description: "Maximum storage space used by the node"
+          description: "Maximum storage space (in bytes) available for the node in Codex's local repository."
         quotaUsedBytes:
           type: integer
           format: int64
-          description: "Amount of storage space currently in use"
+          description: "Amount of storage space (in bytes) currently used for storing files in Codex's local repository."
         quotaReservedBytes:
           type: integer
           format: int64
-          description: "Amount of storage space reserved"
+          description: "Amount of storage reserved (in bytes) in the Codex's local repository for future use when storage requests will be picked up and hosted by the node using node's availabilities. This does not include the storage currently in use."
 
 servers:
   - url: "http://localhost:8080/api/codex/v1"
@@ -382,12 +458,29 @@ paths:
           description: Invalid CID is specified
         "404":
           description: Content specified by the CID is not found
+        "422":
+          description: The content type is not a valid content type or the filename is not valid
         "500":
           description: Well it was bad-bad
     post:
       summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
       tags: [ Data ]
       operationId: upload
+      parameters:
+        - name: content-type
+          in: header
+          required: false
+          description: The content type of the file. Must be valid.
+          schema:
+            type: string
+            example: "image/png"
+        - name: content-disposition
+          in: header
+          required: false
+          description: The content disposition used to send the filename.
+          schema:
+            type: string
+            example: "attachment; filename=\"codex.png\""
       requestBody:
         content:
           application/octet-stream:
@@ -433,10 +526,36 @@ paths:
           description: Well it was bad-bad
 
   "/data/{cid}/network":
+    post:
+      summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
+      tags: [ Data ]
+      operationId: downloadNetwork
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File to be downloaded."
+      responses:
+        "200":
+          description: Manifest information for download that has been started.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
+  "/data/{cid}/network/stream":
     get:
       summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
       tags: [ Data ]
-      operationId: downloadNetwork
+      operationId: downloadNetworkStream
       parameters:
         - in: path
           name: cid
@@ -459,6 +578,32 @@ paths:
         "500":
           description: Well it was bad-bad
 
+  "/data/{cid}/network/manifest":
+    get:
+      summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
+      tags: [ Data ]
+      operationId: downloadNetworkManifest
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File for which the manifest is to be downloaded."
+      responses:
+        "200":
+          description: Manifest information.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
   "/space":
     get:
       summary: "Gets a summary of the storage space allocation of the node."
@@ -491,7 +636,7 @@ paths:
                   $ref: "#/components/schemas/Slot"
 
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/sales/slots/{slotId}":
     get:
@@ -511,7 +656,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/Slot"
+                $ref: "#/components/schemas/SlotAgent"
 
         "400":
           description: Invalid or missing SlotId
@@ -520,13 +665,13 @@ paths:
           description: Host is not in an active sale for the slot
 
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/sales/availability":
     get:
       summary: "Returns storage that is for sale"
       tags: [ Marketplace ]
-      operationId: getOfferedStorage
+      operationId: getAvailabilities
       responses:
         "200":
           description: Retrieved storage availabilities of the node
@@ -535,11 +680,11 @@ paths:
               schema:
                 type: array
                 items:
-                  $ref: "#/components/schemas/SalesAvailability"
+                  $ref: "#/components/schemas/SalesAvailabilityREAD"
         "500":
           description: Error getting unused availabilities
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
     post:
       summary: "Offers storage for sale"
@@ -564,7 +709,7 @@ paths:
         "500":
           description: Error reserving availability
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
   "/sales/availability/{id}":
     patch:
       summary: "Updates availability"
@@ -597,10 +742,10 @@ paths:
         "500":
           description: Error reserving availability
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/sales/availability/{id}/reservations":
-    patch:
+    get:
       summary: "Get availability's reservations"
       description: Return's list of Reservations for ongoing Storage Requests that the node hosts.
       operationId: getReservations
@@ -628,7 +773,7 @@ paths:
         "500":
           description: Error getting reservations
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/storage/request/{cid}":
     post:
@@ -659,7 +804,7 @@ paths:
         "404":
           description: Request ID not found
         "503":
-          description: Purchasing is unavailable
+          description: Persistence is not enabled
 
   "/storage/purchases":
     get:
@@ -676,7 +821,7 @@ paths:
                 items:
                   type: string
         "503":
-          description: Purchasing is unavailable
+          description: Persistence is not enabled
 
   "/storage/purchases/{id}":
     get:
@@ -702,9 +847,9 @@ paths:
         "404":
           description: Purchase not found
         "503":
-          description: Purchasing is unavailable
+          description: Persistence is not enabled
 
-  "/node/spr":
+  "/spr":
     get:
       summary: "Get Node's SPR"
       operationId: getSPR
@@ -722,7 +867,7 @@ paths:
         "503":
           description: Node SPR not ready, try again later
 
-  "/node/peerid":
+  "/peerid":
     get:
       summary: "Get Node's PeerID"
       operationId: getPeerId
@@ -770,4 +915,4 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/DebugInfo"
+                $ref: "#/components/schemas/DebugInfo"

ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj

@@ -2,9 +2,13 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+  </ItemGroup>
+
 </Project>

ProjectPlugins/CodexPluginPrebuild/Program.cs

@@ -1,5 +1,6 @@
 using System.Security.Cryptography;
 using System.Text;
+using Utils;
 
 public static class Program
 {
@@ -40,32 +41,9 @@ public static class Program
 
     private static string FindCodexPluginFolder()
     {
-        var current = Directory.GetCurrentDirectory();
-
-        while (true)
-        {
-            var localFolders = Directory.GetDirectories(current);
-            var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
-            if (projectPluginsFolders.Length == 1)
-            {
-                return Path.Combine(projectPluginsFolders.Single(), CodexPluginFolderName);
-            }
-            var codexPluginFolders = localFolders.Where(l => l.EndsWith(CodexPluginFolderName)).ToArray();
-            if (codexPluginFolders.Length == 1)
-            {
-                return codexPluginFolders.Single();
-            }
-
-            var parent = Directory.GetParent(current);
-            if (parent == null)
-            {
-                var msg = $"Unable to locate '{CodexPluginFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
-                Console.WriteLine(msg);
-                throw new Exception(msg);
-            }
-
-            current = parent.FullName;
-        }
+        var folder = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin");
+        if (!Directory.Exists(folder)) throw new Exception("CodexPlugin folder not found. Expected: " + folder);
+        return folder;
     }
 
     private static string CreateHash(string openApiFile)

ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/GethPlugin/CoreInterfaceExtensions.cs

@@ -1,4 +1,5 @@
-using Core;
+using BlockchainUtils;
+using Core;
 
 namespace GethPlugin
 {
@@ -9,15 +10,15 @@ namespace GethPlugin
             return Plugin(ci).DeployGeth(setup);
         }
 
-        public static IGethNode WrapGethDeployment(this CoreInterface ci, GethDeployment deployment)
+        public static IGethNode WrapGethDeployment(this CoreInterface ci, GethDeployment deployment, BlockCache blockCache)
         {
-            return Plugin(ci).WrapGethDeployment(deployment);
+            return Plugin(ci).WrapGethDeployment(deployment, blockCache);
         }
 
-        public static IGethNode StartGethNode(this CoreInterface ci, Action<IGethSetup> setup)
+        public static IGethNode StartGethNode(this CoreInterface ci, BlockCache blockCache, Action<IGethSetup> setup)
         {
             var deploy = DeployGeth(ci, setup);
-            return WrapGethDeployment(ci, deploy);
+            return WrapGethDeployment(ci, deploy, blockCache);
         }
 
         private static GethPlugin Plugin(CoreInterface ci)

ProjectPlugins/GethPlugin/EthTokenExtensions.cs

@@ -28,7 +28,13 @@
 
         public override string ToString()
         {
-            return $"{Eth} Eth";
+            var weiOnly = Wei % TokensIntExtensions.WeiPerEth;
+
+            var tokens = new List<string>();
+            if (Eth > 0) tokens.Add($"{Eth} Eth");
+            if (weiOnly > 0) tokens.Add($"{weiOnly} Wei");
+
+            return string.Join(" + ", tokens);
         }
     }
 

ProjectPlugins/GethPlugin/GethContainerRecipe.cs

@@ -6,6 +6,7 @@ namespace GethPlugin
     public class GethContainerRecipe : ContainerRecipeFactory
     {
         public static string DockerImage { get; } = "codexstorage/dist-tests-geth:latest";
+        public static TimeSpan BlockInterval { get; } = TimeSpan.FromSeconds(1.0);
         private const string defaultArgs = "--ipcdisable --syncmode full";
 
         public const string HttpPortTag = "http_port";

ProjectPlugins/GethPlugin/GethNode.cs

@@ -1,11 +1,11 @@
-using Core;
+using BlockchainUtils;
+using Core;
 using KubernetesWorkflow.Types;
 using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using NethereumWorkflow;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace GethPlugin
@@ -34,10 +34,12 @@ namespace GethPlugin
     public class DeploymentGethNode : BaseGethNode, IGethNode
     {
         private readonly ILog log;
+        private readonly BlockCache blockCache;
 
-        public DeploymentGethNode(ILog log, GethDeployment startResult)
+        public DeploymentGethNode(ILog log, BlockCache blockCache, GethDeployment startResult)
         {
             this.log = log;
+            this.blockCache = blockCache;
             StartResult = startResult;
         }
 
@@ -57,10 +59,10 @@ namespace GethPlugin
 
         protected override NethereumInteraction StartInteraction()
         {
-            var address = StartResult.Container.GetAddress(log, GethContainerRecipe.HttpPortTag);
+            var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
             var account = StartResult.Account;
 
-            var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);
+            var creator = new NethereumInteractionCreator(log, blockCache, address.Host, address.Port, account.PrivateKey);
             return creator.CreateWorkflow();
         }
     }
@@ -68,6 +70,7 @@ namespace GethPlugin
     public class CustomGethNode : BaseGethNode, IGethNode
     {
         private readonly ILog log;
+        private readonly BlockCache blockCache;
         private readonly string gethHost;
         private readonly int gethPort;
         private readonly string privateKey;
@@ -75,9 +78,10 @@ namespace GethPlugin
         public GethDeployment StartResult => throw new NotImplementedException();
         public RunningContainer Container => throw new NotImplementedException();
 
-        public CustomGethNode(ILog log, string gethHost, int gethPort, string privateKey)
+        public CustomGethNode(ILog log, BlockCache blockCache, string gethHost, int gethPort, string privateKey)
         {
             this.log = log;
+            this.blockCache = blockCache;
             this.gethHost = gethHost;
             this.gethPort = gethPort;
             this.privateKey = privateKey;
@@ -90,7 +94,7 @@ namespace GethPlugin
 
         protected override NethereumInteraction StartInteraction()
         {
-            var creator = new NethereumInteractionCreator(log, gethHost, gethPort, privateKey);
+            var creator = new NethereumInteractionCreator(log, blockCache, gethHost, gethPort, privateKey);
             return creator.CreateWorkflow();
         }
     }

ProjectPlugins/GethPlugin/GethPlugin.cs

@@ -1,4 +1,5 @@
-using Core;
+using BlockchainUtils;
+using Core;
 
 namespace GethPlugin
 {
@@ -36,10 +37,10 @@ namespace GethPlugin
             return starter.StartGeth(startupConfig);
         }
 
-        public IGethNode WrapGethDeployment(GethDeployment startResult)
+        public IGethNode WrapGethDeployment(GethDeployment startResult, BlockCache blockCache)
         {
             startResult = SerializeGate.Gate(startResult);
-            return starter.WrapGethContainer(startResult);
+            return starter.WrapGethContainer(startResult, blockCache);
         }
     }
 }

ProjectPlugins/GethPlugin/GethPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/GethPlugin/GethStarter.cs

@@ -1,4 +1,5 @@
-using Core;
+using BlockchainUtils;
+using Core;
 using KubernetesWorkflow;
 
 namespace GethPlugin
@@ -41,10 +42,10 @@ namespace GethPlugin
             return new GethDeployment(containers, discoveryPort, httpPort, wsPort, account, pubKey);
         }
 
-        public IGethNode WrapGethContainer(GethDeployment startResult)
+        public IGethNode WrapGethContainer(GethDeployment startResult, BlockCache blockCache)
         {
             startResult = SerializeGate.Gate(startResult);
-            return new DeploymentGethNode(tools.GetLog(), startResult);
+            return new DeploymentGethNode(tools.GetLog(), blockCache, startResult);
         }
 
         private void Log(string msg)

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -6,14 +6,14 @@ namespace MetricsPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray(), scrapeInterval);
         }
 
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets);
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets, scrapeInterval);
         }
 
         public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
@@ -26,19 +26,19 @@ namespace MetricsPlugin
             return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
         {
-            return ci.GetMetricsFor(manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return ci.GetMetricsFor(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            var rc = ci.DeployMetricsCollector(scrapeTargets);
+            var rc = ci.DeployMetricsCollector(scrapeInterval, scrapeTargets);
             return scrapeTargets.Select(t => ci.WrapMetricsCollector(rc, t)).ToArray();
         }
 

ProjectPlugins/MetricsPlugin/MetricsAccess.cs

@@ -7,7 +7,7 @@ namespace MetricsPlugin
     public interface IMetricsAccess : IHasContainer
     {
         string TargetName { get; }
-        Metrics? GetAllMetrics();
+        Metrics GetAllMetrics();
         MetricsSet GetMetric(string metricName);
         MetricsSet GetMetric(string metricName, TimeSpan timeout);
     }
@@ -27,7 +27,7 @@ namespace MetricsPlugin
         public string TargetName { get; }
         public RunningContainer Container => query.RunningContainer;
 
-        public Metrics? GetAllMetrics()
+        public Metrics GetAllMetrics()
         {
             return query.GetAllMetricsForNode(target);
         }
@@ -54,11 +54,10 @@ namespace MetricsPlugin
             }
         }
 
-        private MetricsSet? GetMostRecent(string metricName)
+        private MetricsSet GetMostRecent(string metricName)
         {
             var result = query.GetMostRecent(metricName, target);
-            if (result == null) return null;
-            return result.Sets.LastOrDefault();
+            return result.Sets.Last();
         }
     }
 }

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -31,9 +31,9 @@ namespace MetricsPlugin
         {
         }
 
-        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
+        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets, TimeSpan scrapeInterval)
         {
-            return starter.CollectMetricsFor(scrapeTargets);
+            return starter.CollectMetricsFor(scrapeTargets, scrapeInterval);
         }
 
         public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)

ProjectPlugins/MetricsPlugin/MetricsPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -15,7 +15,7 @@ namespace MetricsPlugin
         {
             RunningContainer = runningContainer;
             log = tools.GetLog();
-            var address = RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag);
+            var address = RunningContainer.GetAddress(PrometheusContainerRecipe.PortTag);
             endpoint = tools
                 .CreateHttp(address.ToString())
                 .CreateEndpoint(address, "/api/v1/");
@@ -23,10 +23,10 @@ namespace MetricsPlugin
 
         public RunningContainer RunningContainer { get; }
 
-        public Metrics? GetMostRecent(string metricName, IMetricsScrapeTarget target)
+        public Metrics GetMostRecent(string metricName, IMetricsScrapeTarget target)
         {
             var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
-            if (response == null) return null;
+            if (response == null) throw new Exception($"Failed to get most recent metric: {metricName}");
 
             var result = new Metrics
             {
@@ -44,19 +44,20 @@ namespace MetricsPlugin
             return result;
         }
 
-        public Metrics? GetMetrics(string metricName)
+        public Metrics GetMetrics(string metricName)
         {
             var response = GetAll(metricName);
-            if (response == null) return null;
+            if (response == null) throw new Exception($"Failed to get metrics by name: {metricName}");
             var result = MapResponseToMetrics(response);
             Log(metricName, result);
             return result;
         }
 
-        public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
+        public Metrics GetAllMetricsForNode(IMetricsScrapeTarget target)
         {
-            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
-            if (response.status != "success") return null;
+            var instanceString = GetInstanceStringForNode(target);
+            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={instanceString}{GetQueryTimeRange()}");
+            if (response.status != "success") throw new Exception($"Failed to get metrics for target: {instanceString}");
             var result = MapResponseToMetrics(response);
             Log(target, result);
             return result;
@@ -80,18 +81,32 @@ namespace MetricsPlugin
         {
             return new Metrics
             {
-                Sets = response.data.result.Select(r =>
-                {
-                    return new MetricsSet
-                    {
-                        Name = r.metric.__name__,
-                        Instance = r.metric.instance,
-                        Values = MapMultipleValues(r.values)
-                    };
-                }).ToArray()
+                Sets = response.data.result.Select(CreateMetricsSet).ToArray()
             };
         }
 
+        private MetricsSet CreateMetricsSet(PrometheusQueryResponseDataResultEntry r)
+        {
+            var result = new MetricsSet
+            {
+                Name = r.metric.__name__,
+                Instance = r.metric.instance,
+                Values = MapMultipleValues(r.values)
+            };
+
+            if (!string.IsNullOrEmpty(r.metric.file) && !string.IsNullOrEmpty(r.metric.line) && !string.IsNullOrEmpty(r.metric.proc))
+            {
+                result.AsyncProfiler = new AsyncProfilerMetrics
+                {
+                    File = r.metric.file,
+                    Line = r.metric.line,
+                    Proc = r.metric.proc
+                };
+            }
+
+            return result;
+        }
+
         private MetricsSetValue[] MapSingleValue(object[] value)
         {
             if (value != null && value.Length > 0)
@@ -220,14 +235,28 @@ namespace MetricsPlugin
     {
         public string Name { get; set; } = string.Empty;
         public string Instance { get; set; } = string.Empty;
+        public AsyncProfilerMetrics? AsyncProfiler { get; set; } = null;
         public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
 
         public override string ToString()
         {
-            return $"{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
+            var prefix = "";
+            if (AsyncProfiler != null)
+            {
+                prefix = $"proc: '{AsyncProfiler.Proc}' in '{AsyncProfiler.File}:{AsyncProfiler.Line}'";
+            }
+
+            return $"{prefix}{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
         }
     }
 
+    public class AsyncProfilerMetrics
+    {
+        public string File { get; set; } = string.Empty;
+        public string Line { get; set; } = string.Empty;
+        public string Proc { get; set; } = string.Empty;
+    }
+
     public class MetricsSetValue
     {
         public DateTime Timestamp { get; set; }
@@ -263,6 +292,10 @@ namespace MetricsPlugin
         public string __name__ { get; set; } = string.Empty;
         public string instance { get; set; } = string.Empty;
         public string job { get; set; } = string.Empty;
+        // Async profiler output.
+        public string? file { get; set; } = null;
+        public string? line { get; set; } = null;
+        public string? proc { get; set; } = null;
     }
 
     public class PrometheusAllNamesResponse

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -16,13 +16,13 @@ namespace MetricsPlugin
             this.tools = tools;
         }
 
-        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets)
+        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
         {
             if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
 
             Log($"Starting metrics server for {targets.Length} targets...");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets, scrapeInterval)));
 
             var workflow = tools.CreateWorkflow();
             var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
@@ -48,12 +48,16 @@ namespace MetricsPlugin
             tools.GetLog().Log(msg);
         }
 
-        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets)
+        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
         {
+            var secs = Convert.ToInt32(scrapeInterval.TotalSeconds);
+            if (secs < 1) throw new Exception("ScrapeInterval can't be < 1s");
+            if (secs > 60) throw new Exception("ScrapeInterval can't be > 60s");
+
             var config = "";
             config += "global:\n";
-            config += "  scrape_interval: 10s\n";
-            config += "  scrape_timeout: 10s\n";
+            config += $"  scrape_interval: {secs}s\n";
+            config += $"  scrape_timeout: {secs}s\n";
             config += "\n";
             config += "scrape_configs:\n";
             config += "  - job_name: services\n";
@@ -80,7 +84,7 @@ namespace MetricsPlugin
     {
         public static string FormatTarget(ILog log, IMetricsScrapeTarget target)
         {
-            var a = target.Container.GetAddress(log, target.MetricsPortTag);
+            var a = target.Container.GetAddress(target.MetricsPortTag);
             var host = a.Host.Replace("http://", "").Replace("https://", "");
             return $"{host}:{a.Port}";
         }

README.md

@@ -2,7 +2,7 @@
 
 This project allows you to write tools and tests that control and interact with container-based applications to form a distributed system in a controlled, reproducible environment.
 
-Dotnet: v7.0  
+Dotnet: v8.0  
 Kubernetes: v1.25.4  
 Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp  
 Nethereum: v4.14.0

SpeedCheckTests/SpeedCheckTests.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+	<ItemGroup>
+		<PackageReference Include="nunit" Version="3.13.3" />
+		<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
+		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+	</ItemGroup>
+
+	<ItemGroup>
+		<ProjectReference Include="..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
+		<ProjectReference Include="..\Tests\DistTestCore\DistTestCore.csproj" />
+		<ProjectReference Include="..\Tests\ExperimentalTests\ExperimentalTests.csproj" />
+	</ItemGroup>
+</Project>

SpeedCheckTests/SpeedTest.cs

@@ -0,0 +1,107 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System.Diagnostics;
+using System.Drawing;
+using Utils;
+
+namespace SpeedCheckTests
+{
+    [TestFixture]
+    public class SpeedTest : CodexDistTest
+    {
+        [Test]
+        public void Symmetric()
+        {
+            // Symmetric: A node always sends a reply to every message it receives.
+            CodexContainerRecipe.DockerImageOverride = "thatbenbierens/nim-codex:blkex-cancelpresence-27-f";
+
+            var uploader = StartCodex(s => s.WithName("SymUploader"));
+            var downloader = StartCodex(s => s.WithName("SymDownloader").WithBootstrapNode(uploader));
+            var timeTaken = PerformTest(uploader, downloader);
+
+            Console.WriteLine($"Symmetric time: {Time.FormatDuration(timeTaken)}");
+
+            Assert.That(timeTaken, Is.LessThan(TimeSpan.FromSeconds(10.0)), 
+                $"Symmetric: Too slow. Expected less than 10 seconds but was: {Time.FormatDuration(timeTaken)}");
+        }
+
+        [Test]
+        public void Asymmetric()
+        {
+            // Asymmetric: A node does not always send a reply when a message is received.
+            CodexContainerRecipe.DockerImageOverride = "thatbenbierens/nim-codex:blkex-cancelpresence-27-s";
+
+            var uploader = StartCodex(s => s.WithName("AsymUploader"));
+            var downloader = StartCodex(s => s.WithName("AsymDownloader").WithBootstrapNode(uploader));
+            var timeTaken = PerformTest(uploader, downloader);
+
+            Console.WriteLine($"Asymmetric time: {Time.FormatDuration(timeTaken)}");
+
+            Assert.That(timeTaken, Is.LessThan(TimeSpan.FromSeconds(10.0)),
+                $"Asymmetric: Too slow. Expected less than 10 seconds but was: {Time.FormatDuration(timeTaken)}");
+        }
+
+        [Test]
+        public void Binary()
+        {
+            // Docker image not used: Here for api check.
+            CodexContainerRecipe.DockerImageOverride = "thatbenbierens/nim-codex:blkex-cancelpresence-27-f";
+
+            var binary = "C:\\Projects\\nim-codex\\build\\codex.exe";
+            if (!File.Exists(binary)) throw new Exception("TODO: Update binary path");
+
+            var uploadInfo = new ProcessStartInfo
+            {
+                FileName = binary,
+                Arguments = "--data-dir=upload_data " +
+                "--api-port=8081 " +
+                "--nat=127.0.0.1 " +
+                "--disc-ip=127.0.0.1 " +
+                "--disc-port=8091 " +
+                "--listen-addrs=/ip4/127.0.0.1/tcp/8071",
+                UseShellExecute = true,
+            };
+
+            var uploadProcess = Process.Start(uploadInfo);
+
+            Thread.Sleep(5000);
+            if (uploadProcess == null || uploadProcess.HasExited) throw new Exception("Node exited.");
+
+            CodexAccess.UploaderOverride = new Address("http://localhost", 8081);
+            var uploader = StartCodex(s => s.WithName("BinaryUploader"));
+            var spr = uploader.GetSpr();
+            
+            var downloadProcess = Process.Start(binary,
+                "--data-dir=download_data " +
+                "--api-port=8082 " +
+                "--nat=127.0.0.1 " +
+                "--disc-ip=127.0.0.1 " +
+                "--disc-port=8092 " +
+                "--listen-addrs=/ip4/127.0.0.1/tcp/8072 " +
+                "--bootstrap-node=" + spr
+            );
+
+            CodexAccess.DownloaderOverride = new Address("http://localhost", 8082);
+            var downloader = StartCodex(s => s.WithName("BinaryDownloader"));
+
+            var timeTaken = PerformTest(uploader, downloader);
+
+            uploadProcess.Kill();
+            downloadProcess.Kill();
+
+            Console.WriteLine($"Binary time: {Time.FormatDuration(timeTaken)}");
+
+            Assert.That(timeTaken, Is.LessThan(TimeSpan.FromSeconds(10.0)),
+                $"Binary: Too slow. Expected less than 10 seconds but was: {Time.FormatDuration(timeTaken)}");
+        }
+
+        private TimeSpan PerformTest(ICodexNode uploader, ICodexNode downloader)
+        {
+            var testFile = GenerateTestFile(100.MB());
+            var contentId = uploader.UploadFile(testFile);
+            var (downloadedFile, timeTaken) = downloader.DownloadContentT(contentId);
+            return timeTaken;
+        }
+    }
+}

Tests/CodexContinuousTests/CodexContinuousTests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
@@ -14,8 +14,8 @@
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
-    <ProjectReference Include="..\CodexTests\CodexTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\ExperimentalTests\ExperimentalTests.csproj" />
   </ItemGroup>
 
 </Project>

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -130,7 +130,7 @@ namespace ContinuousTests
                 var namespaceName = container.RunningPod.StartResult.Cluster.Configuration.KubernetesNamespace;
                 var openingLine =
                     $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().Id}";
-                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart,
+                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(node.GetName()), node.Container, effectiveStart,
                     effectiveEnd, openingLine);
             }
         }

Tests/CodexContinuousTests/StartupChecker.cs

@@ -98,7 +98,7 @@ namespace ContinuousTests
             {
                 cancelToken.ThrowIfCancellationRequested();
 
-                var address = n.Container.GetAddress(log, CodexContainerRecipe.ApiPortTag);
+                var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
                 log.Log($"Checking {n.Container.Name} @ '{address}'...");
 
                 if (EnsureOnline(log, n))

Tests/CodexLongTests/CodexTestsLong.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
@@ -14,8 +14,8 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
-    <ProjectReference Include="..\CodexTests\CodexTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\ExperimentalTests\ExperimentalTests.csproj" />
   </ItemGroup>
 
 </Project>

Tests/CodexReleaseTests/CodexReleaseTests.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="NUnit" Version="3.13.3" />
+	<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
+	<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ExperimentalTests\ExperimentalTests.csproj" />
+  </ItemGroup>
+
+</Project>

Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs

@@ -0,0 +1,47 @@
+using CodexPlugin;
+using CodexTests;
+using FileUtils;
+using NUnit.Framework;
+using System.Diagnostics;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    public class InterruptUploadTest : CodexDistTest
+    {
+        [Test]
+        public void UploadInterruptTest()
+        {
+            var nodes = StartCodex(10);
+
+            var tasks = nodes.Select(n => Task<bool>.Run(() => RunInterruptUploadTest(n)));
+            Task.WaitAll(tasks.ToArray());
+
+            Assert.That(tasks.Select(t => t.Result).All(r => r == true));
+
+            WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), nodes);
+        }
+
+        private bool RunInterruptUploadTest(ICodexNode node)
+        {
+            var file = GenerateTestFile(300.MB());
+
+            var process = StartCurlUploadProcess(node, file);
+
+            Thread.Sleep(500);
+            process.Kill();
+            Thread.Sleep(1000);
+
+            var log = Ci.DownloadLog(node);
+            return !log.GetLinesContaining("Unhandled exception in async proc, aborting").Any();
+        }
+
+        private Process StartCurlUploadProcess(ICodexNode node, TrackedFile file)
+        {
+            var apiAddress = node.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
+            var codexUrl = $"{apiAddress}/api/codex/v1/data";
+            var filePath = file.Filename;
+            return Process.Start("curl", $"-X POST {codexUrl} -H \"Content-Type: application/octet-stream\" -T {filePath}");
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/ManifestOnlyDownloadTest.cs

@@ -0,0 +1,33 @@
+using CodexTests;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class ManifestOnlyDownloadTest : CodexDistTest
+    {
+        [Test]
+        public void ManifestOnlyTest()
+        {
+            var uploader = StartCodex();
+            var downloader = StartCodex(s => s.WithBootstrapNode(uploader));
+
+            var file = GenerateTestFile(2.GB());
+            var size = file.GetFilesize().SizeInBytes;
+            var cid = uploader.UploadFile(file);
+
+            var startSpace = downloader.Space();
+            var localDataset = downloader.DownloadManifestOnly(cid);
+
+            Thread.Sleep(1000);
+
+            var spaceDiff = startSpace.FreeBytes - downloader.Space().FreeBytes;
+
+            Assert.That(spaceDiff, Is.LessThan(64.KB().SizeInBytes));
+
+            Assert.That(localDataset.Cid, Is.EqualTo(cid));
+            Assert.That(localDataset.Manifest.OriginalBytes.SizeInBytes, Is.EqualTo(file.GetFilesize().SizeInBytes));
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/OneClientTest.cs

@@ -1,20 +1,26 @@
 using CodexPlugin;
+using CodexTests;
 using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
 using Utils;
 
-namespace CodexTests.BasicTests
+namespace CodexReleaseTests.DataTests
 {
     [TestFixture]
-    public class OneClientTests : CodexDistTest
+    public class OneClientTest : CodexDistTest
     {
         [Test]
-        public void OneClientTest()
+        public void OneClient()
         {
-            var primary = StartCodex();
+            var node = StartCodex();
 
-            PerformOneClientTest(primary);
+            PerformOneClientTest(node);
 
-            LogNodeStatus(primary);
+            LogNodeStatus(node);
         }
 
         private void PerformOneClientTest(ICodexNode primary)
@@ -23,6 +29,8 @@ namespace CodexTests.BasicTests
 
             var contentId = primary.UploadFile(testFile);
 
+            AssertNodesContainFile(contentId, primary);
+
             var downloadedFile = primary.DownloadContent(contentId);
 
             testFile.AssertIsEqual(downloadedFile);

Tests/CodexReleaseTests/DataTests/StreamlessDownloadTest.cs

@@ -0,0 +1,35 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System.Drawing;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class StreamlessDownloadTest : CodexDistTest
+    {
+        [Test]
+        public void StreamlessTest()
+        {
+            var uploader = StartCodex();
+            var downloader = StartCodex(s => s.WithBootstrapNode(uploader));
+
+            var size = 10.MB();
+            var file = GenerateTestFile(size);
+            var cid = uploader.UploadFile(file);
+
+            var startSpace = downloader.Space();
+            var start = DateTime.UtcNow;
+            var localDataset = downloader.DownloadStreamlessWait(cid, size);
+
+            Assert.That(localDataset.Cid, Is.EqualTo(cid));
+            Assert.That(localDataset.Manifest.OriginalBytes.SizeInBytes, Is.EqualTo(file.GetFilesize().SizeInBytes));
+
+            // Stop the uploader node and verify that the downloader has the data.
+            uploader.Stop(waitTillStopped: true);
+            var downloaded = downloader.DownloadContent(cid);
+            file.AssertIsEqual(downloaded);
+        }
+    }
+}