investigate wip

* Fix job status check

* Add a delay before get Kubernetes job status

.gitattributes

@@ -0,0 +1,10 @@
+# Set default behavior to automatically normalize line endings.
+* text=auto
+
+# Force bash scripts to always use lf line endings so that if a repo is accessed
+# in Unix via a file share from Windows, the scripts will work.
+*.sh text eol=lf
+
+# Likewise, force cmd and batch scripts to always use crlf
+*.cmd text eol=crlf
+*.bat text eol=crlf

.github/workflows/run-continuous-tests.yaml

@@ -80,7 +80,7 @@ env:
   TESTS_TARGET_DURATION: 2d
   TESTS_FILTER: ""
   TESTS_CLEANUP: true
-  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  JOB_MANIFEST: docker/job-continuous-tests.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
   KUBE_VERSION: v1.28.2
 
@@ -146,7 +146,7 @@ jobs:
         if: false
         run: |
           # Variables
-          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to run a node
           duration=600
           namespace="${{ env.NAMESPACE }}"
           pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")

.github/workflows/run-dist-tests.yaml

@@ -32,7 +32,7 @@ env:
   NAMEPREFIX: d-tests-runner
   NAMESPACE: default
   COMMAND: dotnet test Tests/CodexTests
-  JOB_MANIFEST: docker/dist-tests-job.yaml
+  JOB_MANIFEST: docker/job-dist-tests.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
   KUBE_VERSION: v1.28.2
 

.github/workflows/run-release-tests.yaml

@@ -0,0 +1,121 @@
+name: Run Release Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+      workflow_source:
+        description: Workflow source
+        required: false
+        type: string
+        default: ''
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
+  TEST_TYPE: release-tests
+  NAMEPREFIX: r-tests
+  NAMESPACE: default
+  JOB_MANIFEST: docker/job-release-tests.yaml
+  COMMAND: dotnet test Tests/CodexReleaseTests
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.30.5
+
+
+jobs:
+  run_tests:
+    name: Run Release Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
+          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'${COMMAND}'"')" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job to run tests
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}"
+          echo "Runner pod: $(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}')"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+
+      - name: Show Runner logs
+        run: |
+          # Variables
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          namespace="${{ env.NAMESPACE }}"
+          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath='{.items[0].metadata.name}')
+
+          # Check Pod status
+          WAIT=120
+          SECONDS=0
+          sleep=1
+          while (( SECONDS < WAIT )); do
+            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}')
+            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in the $phase state - Get the logs"; break; } || { echo "Pod $pod is in the $phase state - Retry in $sleep second(s) / $((WAIT - SECONDS))"; }
+            sleep $sleep
+          done
+
+          # Get logs
+          while [[ $(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}') == "Running" ]]; do
+            echo "Show ${pod} logs ..."
+            kubectl logs $pod -n $namespace -f || true
+            sleep 1
+          done
+
+      - name: Set workflow status from job status
+        run: |
+          sleep 5
+          job_status=$(kubectl get jobs ${{ env.NAMEPREFIX }} -n ${{ env.NAMESPACE }} -o jsonpath='{.status.conditions[0].type}')
+          echo "Job status: $job_status"
+          if [[ "${job_status}" != "Complete" ]]; then exit 1; fi

Framework/ArgsUniform/ArgsUniform.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/Core/Core.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/DiscordRewards/DiscordRewards.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -3,7 +3,8 @@
     public class GiveRewardsCommand
     {
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
-        public string[] EventsOverview { get; set; } = Array.Empty<string>();
+        public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty<ChainEventMessage>();
+        public string[] Errors { get; set; } = Array.Empty<string>();
 
         public bool HasAny()
         {
@@ -16,4 +17,10 @@
         public ulong RewardId { get; set; }
         public string[] UserAddresses { get; set; } = Array.Empty<string>();
     }
+
+    public class ChainEventMessage
+    {
+        public ulong BlockNumber { get; set; }
+        public string Message { get; set; } = string.Empty;
+    }
 }

Framework/FileUtils/FileUtils.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/GethConnector/GethConnector.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 
@@ -18,18 +19,29 @@ namespace GethConnector
                 return null;
             }
 
+            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+
+            var config = GetCodexMarketplaceConfig(gethNode, GethInput.MarketplaceAddress);
+
             var contractsDeployment = new CodexContractsDeployment(
+                config: config,
                 marketplaceAddress: GethInput.MarketplaceAddress,
                 abi: GethInput.ABI,
                 tokenAddress: GethInput.TokenAddress
             );
 
-            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
             var contracts = new CodexContractsAccess(log, gethNode, contractsDeployment);
 
             return new GethConnector(gethNode, contracts);
         }
 
+        private static MarketplaceConfig GetCodexMarketplaceConfig(IGethNode gethNode, string marketplaceAddress)
+        {
+            var func = new ConfigurationFunctionBase();
+            var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
+            return response.ReturnValue1;
+        }
+
         private GethConnector(IGethNode gethNode, ICodexContracts codexContracts)
         {
             GethNode = gethNode;

Framework/GethConnector/GethConnector.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/KubernetesWorkflow/DownloadedLog.cs

@@ -6,6 +6,7 @@ namespace KubernetesWorkflow
     {
         string ContainerName { get; }
 
+        void IterateLines(Action<string> action);
         void IterateLines(Action<string> action, params string[] thatContain);
         string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
@@ -25,58 +26,39 @@ namespace KubernetesWorkflow
 
         public string ContainerName { get; }
 
-        public void IterateLines(Action<string> action, params string[] thatContain)
+        public void IterateLines(Action<string> action)
         {
             using var file = File.OpenRead(logFile.FullFilename);
             using var streamReader = new StreamReader(file);
 
             var line = streamReader.ReadLine();
             while (line != null)
+            {
+                action(line);
+                line = streamReader.ReadLine();
+            }
+        }
+
+        public void IterateLines(Action<string> action, params string[] thatContain)
+        {
+            IterateLines(line =>
             {
                 if (thatContain.All(line.Contains))
                 {
                     action(line);
                 }
-                line = streamReader.ReadLine();
-            }
+            });
         }
 
         public string[] GetLinesContaining(string expectedString)
         {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-            var lines = new List<string>();
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (line.Contains(expectedString))
-                {
-                    lines.Add(line);
-                }
-                line = streamReader.ReadLine();
-            }
-
-            return lines.ToArray(); ;
+            return FindLinesThatContain([expectedString]);
         }
 
         public string[] FindLinesThatContain(params string[] tags)
         {
             var result = new List<string>();
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (tags.All(line.Contains))
-                {
-                    result.Add(line);
-                }
-
-                line = streamReader.ReadLine();
-            }
-
+            IterateLines(result.Add, tags);
             return result.ToArray();
         }
 

Framework/KubernetesWorkflow/K8sNameUtils.cs

@@ -24,8 +24,8 @@
                 .Replace("]", "-")
                 .Replace(",", "-");
 
-            result = result.Trim('-');
             if (result.Length > maxLength) result = result.Substring(0, maxLength);
+            result = result.Trim('-');
 
             return result;
         }

Framework/KubernetesWorkflow/KubernetesWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>KubernetesWorkflow</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -24,13 +24,12 @@ namespace KubernetesWorkflow.Types
         [JsonIgnore]
         public RunningPod RunningPod { get; internal set; } = null!;
 
-        public Address GetAddress(ILog log, string portTag)
+        public Address GetAddress(string portTag)
         {
             var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
             if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);
 
             var select = SelectAddress(addresses);
-            log.Debug($"Container '{Name}' selected for tag '{portTag}' address: '{select}'");
             return select.Address;
         }
 

Framework/Logging/Logging.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Logging</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -29,7 +29,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();
-            if (moment <= bounds.Genesis.Utc) return null;
+            if (moment < bounds.Genesis.Utc) return null;
+            if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
             if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
 
             return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
@@ -38,7 +39,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetLowestBlockNumberAfter(DateTime moment)
         {
             bounds.Initialize();
-            if (moment >= bounds.Current.Utc) return null;
+            if (moment > bounds.Current.Utc) return null;
+            if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber;
             if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
 
             return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;

Framework/NethereumWorkflow/NethereumWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	<RootNamespace>NethereumWorkflow</RootNamespace>
 	<ImplicitUsings>enable</ImplicitUsings>
 	<Nullable>enable</Nullable>

Framework/OverwatchTranscript/EventBucketReader.cs

@@ -7,6 +7,7 @@ namespace OverwatchTranscript
     public interface IFinalizedBucket
     {
         bool IsEmpty { get; }
+        void Update();
         DateTime? SeeTopUtc();
         BucketTop? TakeTop();
     }
@@ -28,7 +29,8 @@ namespace OverwatchTranscript
         private readonly string bucketFile;
         private readonly ConcurrentQueue<BucketTop> topQueue = new ConcurrentQueue<BucketTop>();
         private readonly AutoResetEvent itemDequeued = new AutoResetEvent(false);
-        private bool stopping;
+        private readonly AutoResetEvent itemEnqueued = new AutoResetEvent(false);
+        private bool sourceIsEmpty;
 
         public EventBucketReader(ILog log, string bucketFile)
         {
@@ -42,34 +44,38 @@ namespace OverwatchTranscript
         
         public bool IsEmpty { get; private set; }
 
+        public void Update()
+        {
+            if (IsEmpty) return;
+            while (topQueue.Count == 0)
+            {
+                UpdateIsEmpty();
+                if (IsEmpty) return;
+
+                itemDequeued.Set();
+                itemEnqueued.WaitOne(200);
+            }
+        }
+
         public DateTime? SeeTopUtc()
         {
             if (IsEmpty) return null;
-            while (true)
+            if (topQueue.TryPeek(out BucketTop? top))
             {
-                UpdateIsEmpty();
-                if (IsEmpty) return null;
-                if (topQueue.TryPeek(out BucketTop? top))
-                {
-                    return top.Utc;
-                }
+                return top.Utc;
             }
+            return null;
         }
 
         public BucketTop? TakeTop()
         {
             if (IsEmpty) return null;
-
-            while (true)
+            if (topQueue.TryDequeue(out BucketTop? top))
             {
-                UpdateIsEmpty();
-                if (IsEmpty) return null;
-                if (topQueue.TryDequeue(out BucketTop? top))
-                {
-                    itemDequeued.Set();
-                    return top;
-                }
+                itemDequeued.Set();
+                return top;
             }
+            return null;
         }
 
         private void ReadBucket()
@@ -85,23 +91,25 @@ namespace OverwatchTranscript
                     if (top != null)
                     {
                         topQueue.Enqueue(top);
+                        itemEnqueued.Set();
                     }
                     else
                     {
-                        stopping = true;
+                        sourceIsEmpty = true;
+                        UpdateIsEmpty();
                         return;
                     }
                 }
 
                 itemDequeued.Reset();
-                itemDequeued.WaitOne();
+                itemDequeued.WaitOne(5000);
             }
         }
 
         private void UpdateIsEmpty()
         {
-            var empty = stopping && topQueue.IsEmpty;
-            if (!IsEmpty && empty)
+            var allEmpty = sourceIsEmpty && topQueue.IsEmpty;
+            if (!IsEmpty && allEmpty)
             {
                 File.Delete(bucketFile);
                 IsEmpty = true;

Framework/OverwatchTranscript/MomentReferenceBuilder.cs

@@ -24,6 +24,8 @@ namespace OverwatchTranscript
             log.Debug($"Building references for {buckets.Count} buckets.");
             while (buckets.Any())
             {
+                foreach (var b in buckets) b.Update();
+
                 buckets.RemoveAll(b => b.IsEmpty);
                 if (!buckets.Any()) break;
 

Framework/OverwatchTranscript/OverwatchTranscript.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/Utils/BlockInterval.cs

@@ -15,12 +15,13 @@
                 To = from;
             }
             TimeRange = timeRange;
+            NumberOfBlocks = (To - From) + 1;
         }
 
         public ulong From { get; }
         public ulong To { get; }
         public TimeRange TimeRange { get; }
-        public ulong NumberOfBlocks => To - From;
+        public ulong NumberOfBlocks { get; }
 
         public override string ToString()
         {

Framework/Utils/PluginPathUtils.cs

@@ -0,0 +1,41 @@
+namespace Utils
+{
+    public static class PluginPathUtils
+    {
+        private const string ProjectPluginsFolderName = "ProjectPlugins";
+        private static string projectPluginsDir = string.Empty;
+
+        public static string ProjectPluginsDir
+        {
+            get
+            {
+                if (string.IsNullOrEmpty(projectPluginsDir)) projectPluginsDir = FindProjectPluginsDir();
+                return projectPluginsDir;
+            }
+        }
+
+        private static string FindProjectPluginsDir()
+        {
+            var current = Directory.GetCurrentDirectory();
+            while (true)
+            {
+                var localFolders = Directory.GetDirectories(current);
+                var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
+                if (projectPluginsFolders.Length == 1)
+                {
+                    return projectPluginsFolders.Single();
+                }
+
+                var parent = Directory.GetParent(current);
+                if (parent == null)
+                {
+                    var msg = $"Unable to locate '{ProjectPluginsFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
+                    Console.WriteLine(msg);
+                    throw new Exception(msg);
+                }
+
+                current = parent.FullName;
+            }
+        }
+    }
+}

Framework/Utils/RandomUtils.cs

@@ -3,24 +3,41 @@
     public static class RandomUtils
     {
         private static readonly Random random = new Random();
+        private static readonly object @lock = new object();
+
+        public static T GetOneRandom<T>(this T[] items)
+        {
+            lock (@lock)
+            {
+                var i = random.Next(0, items.Length);
+                var result = items[i];
+                return result;
+            }
+        }
 
         public static T PickOneRandom<T>(this List<T> remainingItems)
         {
-            var i = random.Next(0, remainingItems.Count);
-            var result = remainingItems[i];
-            remainingItems.RemoveAt(i);
-            return result;
+            lock (@lock)
+            {
+                var i = random.Next(0, remainingItems.Count);
+                var result = remainingItems[i];
+                remainingItems.RemoveAt(i);
+                return result;
+            }
         }
 
         public static T[] Shuffled<T>(T[] items)
         {
-            var result = new List<T>();
-            var source = items.ToList();
-            while (source.Any())
+            lock (@lock)
             {
-                result.Add(RandomUtils.PickOneRandom(source));
+                var result = new List<T>();
+                var source = items.ToList();
+                while (source.Any())
+                {
+                    result.Add(RandomUtils.PickOneRandom(source));
+                }
+                return result.ToArray();
             }
-            return result.ToArray();
         }
     }
 }

Framework/Utils/Retry.cs

@@ -71,6 +71,10 @@
                         task();
                         return;
                     }
+                    catch (OperationCanceledException)
+                    {
+                        return;
+                    }
                     catch (Exception ex)
                     {
                         var failure = CaptureFailure(ex);

Framework/Utils/Utils.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Utils</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin.Marketplace;
+using System.Collections.Generic;
 using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
@@ -12,7 +13,8 @@ namespace CodexContractsPlugin.ChainMonitor
             RequestCancelledEventDTO[] cancelled,
             RequestFailedEventDTO[] failed,
             SlotFilledEventDTO[] slotFilled,
-            SlotFreedEventDTO[] slotFreed
+            SlotFreedEventDTO[] slotFreed,
+            SlotReservationsFullEventDTO[] slotReservationsFull
             )
         {
             BlockInterval = blockInterval;
@@ -22,6 +24,9 @@ namespace CodexContractsPlugin.ChainMonitor
             Failed = failed;
             SlotFilled = slotFilled;
             SlotFreed = slotFreed;
+            SlotReservationsFull = slotReservationsFull;
+
+            All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull);
         }
 
         public BlockInterval BlockInterval { get; }
@@ -31,21 +36,8 @@ namespace CodexContractsPlugin.ChainMonitor
         public RequestFailedEventDTO[] Failed { get; }
         public SlotFilledEventDTO[] SlotFilled { get; }
         public SlotFreedEventDTO[] SlotFreed { get; }
-
-        public IHasBlock[] All
-        {
-            get
-            {
-                var all = new List<IHasBlock>();
-                all.AddRange(Requests);
-                all.AddRange(Fulfilled);
-                all.AddRange(Cancelled);
-                all.AddRange(Failed);
-                all.AddRange(SlotFilled);
-                all.AddRange(SlotFreed);
-                return all.ToArray();
-            }
-        }
+        public SlotReservationsFullEventDTO[] SlotReservationsFull { get; }
+        public IHasBlock[] All { get; }
 
         public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
         {
@@ -66,8 +58,19 @@ namespace CodexContractsPlugin.ChainMonitor
                 events.GetRequestCancelledEvents(),
                 events.GetRequestFailedEvents(),
                 events.GetSlotFilledEvents(),
-                events.GetSlotFreedEvents()
+                events.GetSlotFreedEvents(),
+                events.GetSlotReservationsFull()
             );
         }
+
+        private T[] ConcatAll<T>(params T[][] arrays)
+        {
+            var result = Array.Empty<T>();
+            foreach (var array in arrays)
+            {
+                result = result.Concat(array).ToArray();
+            }
+            return result;
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -16,6 +16,9 @@ namespace CodexContractsPlugin.ChainMonitor
         void OnRequestFailed(RequestEvent requestEvent);
         void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
         void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
+        void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
+
+        void OnError(string msg);
     }
 
     public class RequestEvent
@@ -48,30 +51,36 @@ namespace CodexContractsPlugin.ChainMonitor
         public TimeRange TotalSpan { get; private set; }
         public IChainStateRequest[] Requests => requests.ToArray();
 
-        public void Update()
+        public int Update()
         {
-            Update(DateTime.UtcNow);
+            return Update(DateTime.UtcNow);
         }
 
-        public void Update(DateTime toUtc)
+        public int Update(DateTime toUtc)
         {
             var span = new TimeRange(TotalSpan.To, toUtc);
             var events = ChainEvents.FromTimeRange(contracts, span);
             Apply(events);
 
             TotalSpan = new TimeRange(TotalSpan.From, span.To);
+            return events.All.Length;
         }
 
         private void Apply(ChainEvents events)
         {
             if (events.BlockInterval.TimeRange.From < TotalSpan.From)
-                throw new Exception("Attempt to update ChainState with set of events from before its current record.");
+            {
+                var msg = "Attempt to update ChainState with set of events from before its current record.";
+                handler.OnError(msg);
+                throw new Exception(msg);
+            }
 
-            log.Log($"ChainState updating: {events.BlockInterval}");
+            log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
 
             // Run through each block and apply the events to the state in order.
             var span = events.BlockInterval.TimeRange.Duration;
             var numBlocks = events.BlockInterval.NumberOfBlocks;
+            if (numBlocks == 0) return;
             var spanPerBlock = span / numBlocks;
 
             var eventUtc = events.BlockInterval.TimeRange.From;
@@ -108,7 +117,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFulfilledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
             handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
@@ -116,7 +125,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestCancelledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
             handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
@@ -124,7 +133,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(RequestFailedEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.UpdateState(@event.Block.BlockNumber, RequestState.Failed);
             handler.OnRequestFailed(new RequestEvent(@event.Block, r));
@@ -132,7 +141,7 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFilledEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
@@ -141,13 +150,21 @@ namespace CodexContractsPlugin.ChainMonitor
 
         private void ApplyEvent(SlotFreedEventDTO @event)
         {
-            var r = FindRequest(@event.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
             r.Hosts.RemoveHost((int)@event.SlotIndex);
             r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
             handler.OnSlotFreed(new RequestEvent(@event.Block, r), @event.SlotIndex);
         }
 
+        private void ApplyEvent(SlotReservationsFullEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.Log($"[{@event.Block.BlockNumber}] SlotReservationsFull (slotIndex:{@event.SlotIndex})");
+            handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
+        }
+
         private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
         {
             foreach (var r in requests)
@@ -161,10 +178,23 @@ namespace CodexContractsPlugin.ChainMonitor
             }
         }
 
-        private ChainStateRequest? FindRequest(byte[] requestId)
+        private ChainStateRequest? FindRequest(IHasRequestId request)
         {
-            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
-            if (r == null) log.Log("Unable to find request by ID!");
+            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
+            if (r == null)
+            {
+                var blockNumber = "unknown";
+                if (request is IHasBlock blk)
+                {
+                    blockNumber = blk.Block.BlockNumber.ToString();
+                }
+
+                var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
+                    $"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
+
+                log.Error(msg);
+                handler.OnError(msg);
+            }
             return r;
         }
 

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -1,10 +1,5 @@
 using GethPlugin;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Numerics;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
@@ -51,5 +46,15 @@ namespace CodexContractsPlugin.ChainMonitor
         {
             foreach (var handler in Handlers) handler.OnSlotFreed(requestEvent, slotIndex);
         }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotReservationsFull(requestEvent, slotIndex);
+        }
+
+        public void OnError(string msg)
+        {
+            foreach (var handler in Handlers) handler.OnError(msg);
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -32,5 +32,13 @@ namespace CodexContractsPlugin.ChainMonitor
         public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
         {
         }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public void OnError(string msg)
+        {
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -2,6 +2,7 @@
 using GethPlugin;
 using Logging;
 using Nethereum.ABI;
+using Nethereum.Hex.HexConvertors.Extensions;
 using Nethereum.Util;
 using NethereumWorkflow;
 using Newtonsoft.Json;
@@ -24,6 +25,7 @@ namespace CodexContractsPlugin
         ICodexContractsEvents GetEvents(BlockInterval blockInterval);
         EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
         RequestState GetRequestState(Request request);
+        void WaitUntilNextPeriod();
     }
 
     [JsonConverter(typeof(StringEnumConverter))]
@@ -114,6 +116,15 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
+        public void WaitUntilNextPeriod()
+        {
+            log.Log("Waiting until next proof period...");
+            var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
+            var periodSeconds = (int)Deployment.Config.Proofs.Period;
+            var secondsLeft = now % periodSeconds;
+            Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1));
+        }
+
         private ContractInteractions StartInteraction()
         {
             return new ContractInteractions(log, gethNode);

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -19,7 +19,7 @@ namespace CodexContractsPlugin
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 
-            var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
+            var address = config.GethNode.StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
 
             SetSchedulingAffinity(notIn: "false");
 

ProjectPlugins/CodexContractsPlugin/CodexContractsDeployment.cs

@@ -1,14 +1,18 @@
-namespace CodexContractsPlugin
+using CodexContractsPlugin.Marketplace;
+
+namespace CodexContractsPlugin
 {
     public class CodexContractsDeployment
     {
-        public CodexContractsDeployment(string marketplaceAddress, string abi, string tokenAddress)
+        public CodexContractsDeployment(MarketplaceConfig config, string marketplaceAddress, string abi, string tokenAddress)
         {
+            Config = config;
             MarketplaceAddress = marketplaceAddress;
             Abi = abi;
             TokenAddress = tokenAddress;
         }
 
+        public MarketplaceConfig Config { get; }
         public string MarketplaceAddress { get; }
         public string Abi { get; }
         public string TokenAddress { get; }

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -1,6 +1,7 @@
 using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
+using Nethereum.Contracts;
 using Nethereum.Hex.HexTypes;
 using NethereumWorkflow.BlockUtils;
 using Utils;
@@ -16,6 +17,7 @@ namespace CodexContractsPlugin
         RequestFailedEventDTO[] GetRequestFailedEvents();
         SlotFilledEventDTO[] GetSlotFilledEvents();
         SlotFreedEventDTO[] GetSlotFreedEvents();
+        SlotReservationsFullEventDTO[] GetSlotReservationsFull();
     }
 
     public class CodexContractsEvents : ICodexContractsEvents
@@ -38,49 +40,32 @@ namespace CodexContractsPlugin
         {
             var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
             var i = new ContractInteractions(log, gethNode);
-            return events
-            .Select(e =>
+            return events.Select(e =>
             {
-                        var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
-                        var result = requestEvent.ReturnValue1;
-                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                        result.RequestId = e.Event.RequestId;
-                        return result;
-                    })
-                    .ToArray();
+                var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
+                var result = requestEvent.ReturnValue1;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.RequestId = e.Event.RequestId;
+                return result;
+            }).ToArray();
         }
 
         public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()
         {
             var events = gethNode.GetEvents<RequestFulfilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public RequestCancelledEventDTO[] GetRequestCancelledEvents()
         {
             var events = gethNode.GetEvents<RequestCancelledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public RequestFailedEventDTO[] GetRequestFailedEvents()
         {
             var events = gethNode.GetEvents<RequestFailedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public SlotFilledEventDTO[] GetSlotFilledEvents()
@@ -98,12 +83,20 @@ namespace CodexContractsPlugin
         public SlotFreedEventDTO[] GetSlotFreedEvents()
         {
             var events = gethNode.GetEvents<SlotFreedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        public SlotReservationsFullEventDTO[] GetSlotReservationsFull()
+        {
+            var events = gethNode.GetEvents<SlotReservationsFullEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
+        {
+            var result = e.Event;
+            result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+            return result;
         }
 
         private BlockTimeEntry GetBlock(ulong number)

ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -4,6 +4,7 @@ using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
+using Newtonsoft.Json;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -34,6 +35,7 @@ namespace CodexContractsPlugin
             try
             {
                 var result = DeployContract(container, workflow, gethNode);
+
                 workflow.Stop(containers, waitTillStopped: false);
                 Log("Container stopped.");
                 return result;
@@ -75,9 +77,20 @@ namespace CodexContractsPlugin
 
             Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract));
 
-            Log("Synced. Codex SmartContracts deployed.");
+            Log("Synced. Codex SmartContracts deployed. Getting configuration...");
 
-            return new CodexContractsDeployment(marketplaceAddress, abi, tokenAddress);
+            var config = GetMarketplaceConfiguration(marketplaceAddress, gethNode);
+
+            Log("Got config: " + JsonConvert.SerializeObject(config));
+
+            return new CodexContractsDeployment(config, marketplaceAddress, abi, tokenAddress);
+        }
+
+        private MarketplaceConfig GetMarketplaceConfiguration(string marketplaceAddress, IGethNode gethNode)
+        {
+            var func = new ConfigurationFunctionBase();
+            var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
+            return response.ReturnValue1;
         }
 
         private void EnsureCompatbility(string abi, string bytecode)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -23,9 +23,8 @@ namespace CodexContractsPlugin
         public string GetTokenAddress(string marketplaceAddress)
         {
             log.Debug(marketplaceAddress);
-            var function = new GetTokenFunction();
-
-            return gethNode.Call<GetTokenFunction, string>(marketplaceAddress, function);
+            var function = new TokenFunctionBase();
+            return gethNode.Call<TokenFunctionBase, string>(marketplaceAddress, function);
         }
 
         public string GetTokenName(string tokenAddress)
@@ -111,11 +110,6 @@ namespace CodexContractsPlugin
         }
     }
 
-    [Function("token", "address")]
-    public class GetTokenFunction : FunctionMessage
-    {
-    }
-
     [Function("name", "string")]
     public class GetTokenNameFunction : FunctionMessage
     {

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -10,7 +10,12 @@ namespace CodexContractsPlugin.Marketplace
         BlockTimeEntry Block { get; set; }
     }
 
-    public partial class Request : RequestBase, IHasBlock
+    public interface IHasRequestId
+    {
+        byte[] RequestId { get; set; }
+    }
+
+    public partial class Request : RequestBase, IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
@@ -28,32 +33,38 @@ namespace CodexContractsPlugin.Marketplace
         }
     }
 
-    public partial class RequestFulfilledEventDTO : IHasBlock
+    public partial class RequestFulfilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestCancelledEventDTO : IHasBlock
+    public partial class RequestCancelledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestFailedEventDTO : IHasBlock
+    public partial class RequestFailedEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class SlotFilledEventDTO : IHasBlock
+    public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
     }
 
-    public partial class SlotFreedEventDTO : IHasBlock
+    public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
+
+    public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }

ProjectPlugins/CodexContractsPlugin/SelfUpdater.cs

@@ -1,4 +1,6 @@
-namespace CodexContractsPlugin
+using Utils;
+
+namespace CodexContractsPlugin
 {
     public class SelfUpdater
     {
@@ -41,24 +43,10 @@
 
         private string GetMarketplaceFilePath()
         {
-            var here = Directory.GetCurrentDirectory();
-            while (true)
-            {
-                var path = GetMarketplaceFile(here);
-                if (path != null) return path;
-
-                var parent = Directory.GetParent(here);
-                var up = parent?.FullName;
-                if (up == null || up == here) throw new Exception("Unable to locate ProjectPlugins folder. Unable to update contracts.");
-                here = up;
-            }
-        }
-
-        private string? GetMarketplaceFile(string root)
-        {
-            var path = Path.Combine(root, "ProjectPlugins", "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
-            if (File.Exists(path)) return path;
-            return null;
+            var projectPluginDir = PluginPathUtils.ProjectPluginsDir;
+            var path = Path.Combine(projectPluginDir, "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
+            if (!File.Exists(path)) throw new Exception("Marketplace file not found. Expected: " + path);
+            return path;
         }
 
         private string GenerateContent(string abi, string bytecode)

ProjectPlugins/CodexContractsPlugin/TestTokenExtensions.cs

@@ -46,6 +46,16 @@ namespace CodexContractsPlugin
             return new TestToken(a.TstWei + b.TstWei);
         }
 
+        public static TestToken operator -(TestToken a, TestToken b)
+        {
+            return new TestToken(a.TstWei - b.TstWei);
+        }
+
+        public static TestToken operator *(TestToken a, int b)
+        {
+            return new TestToken(a.TstWei * b);
+        }
+
         public static bool operator <(TestToken a, TestToken b)
         {
             return a.TstWei < b.TstWei;

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -3,13 +3,14 @@ using KubernetesWorkflow.Types;
 using Logging;
 using System.Security.Cryptography;
 using System.Text;
+using Utils;
 
 namespace CodexPlugin
 {
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "67-76-AB-FC-54-4F-EB-81-F5-E4-F8-27-DF-82-92-41-63-A5-EA-1B-17-14-0C-BE-20-9C-B3-DF-CE-E4-AA-38";
+        private const string OpenApiYamlHash = "34-B5-DA-26-40-76-B8-D8-8E-7D-9C-17-85-C6-B0-63-55-8D-C6-01-0B-96-BB-7C-BD-53-E5-32-07-ED-29-92";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 
@@ -21,8 +22,9 @@ namespace CodexPlugin
 
         private const string Failure =
             "Codex API compatibility check failed! " +
-            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. Please update the openapi.yaml in " +
-            "'ProjectPlugins/CodexPlugin' and rebuild this project. If you wish to disable API compatibility checking, please set " +
+            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. The openapi.yaml in " +
+            "'ProjectPlugins/CodexPlugin' has been overwritten with the container one. " +
+            "Please and rebuild this project. If you wish to disable API compatibility checking, please set " +
             $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/CodexPlugin/ApiChecker.cs'.";
 
         private static bool checkPassed = false;
@@ -71,10 +73,23 @@ namespace CodexPlugin
                 return;
             }
 
+            OverwriteOpenApiYaml(containerApi);
+
             log.Error(Failure);
             throw new Exception(Failure);
         }
 
+        private void OverwriteOpenApiYaml(string containerApi)
+        {
+            Log("API compatibility check failed. Updating CodexPlugin...");
+            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin", "openapi.yaml");
+            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexPlugin/openapi.yaml. Expected: " + openApiFilePath);
+
+            File.Delete(openApiFilePath);
+            File.WriteAllText(openApiFilePath, containerApi);
+            Log("CodexPlugin/openapi.yaml has been updated.");
+        }
+
         private string Hash(string file)
         {
             var fileBytes = Encoding.ASCII.GetBytes(file

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -32,6 +32,22 @@ namespace CodexPlugin
             return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
         }
 
+        public string GetSpr()
+        {
+            return CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                var json = endpoint.HttpGetString("spr");
+                var response = JsonConvert.DeserializeObject<SprResponse>(json);
+                return response!.Spr;
+            });
+        }
+
+        private class SprResponse
+        {
+            public string Spr { get; set; } = string.Empty;
+        }
+
         public DebugPeer GetDebugPeer(string peerId)
         {
             // Cannot use openAPI: debug/peer endpoint is not specified there.
@@ -63,23 +79,35 @@ namespace CodexPlugin
             });
         }
 
-        public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
+        public string UploadFile(UploadInput uploadInput, Action<Failure> onFailure)
         {
             return OnCodex(
-                api => api.UploadAsync(fileStream),
+                api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream),
                 CreateRetryConfig(nameof(UploadFile), onFailure));
         }
 
         public Stream DownloadFile(string contentId, Action<Failure> onFailure)
         {
             var fileResponse = OnCodex(
-                api => api.DownloadNetworkAsync(contentId),
+                api => api.DownloadNetworkStreamAsync(contentId),
                 CreateRetryConfig(nameof(DownloadFile), onFailure));
 
             if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
             return fileResponse.Stream;
         }
 
+        public LocalDataset DownloadStreamless(ContentId cid)
+        {
+            var response = OnCodex(api => api.DownloadNetworkAsync(cid.Id));
+            return mapper.Map(response);
+        }
+
+        public LocalDataset DownloadManifestOnly(ContentId cid)
+        {
+            var response = OnCodex(api => api.DownloadNetworkManifestAsync(cid.Id));
+            return mapper.Map(response);
+        }
+
         public LocalDatasetList LocalFiles()
         {
             return mapper.Map(OnCodex(api => api.ListDataAsync()));
@@ -88,19 +116,25 @@ namespace CodexPlugin
         public StorageAvailability SalesAvailability(StorageAvailability request)
         {
             var body = mapper.Map(request);
-            var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
+            var read = OnCodex(api => api.OfferStorageAsync(body));
             return mapper.Map(read);
         }
 
+        public StorageAvailability[] GetAvailabilities()
+        {
+            var collection = OnCodex(api => api.GetAvailabilitiesAsync());
+            return mapper.Map(collection);
+        }
+
         public string RequestStorage(StoragePurchaseRequest request)
         {
             var body = mapper.Map(request);
-            return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
+            return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
         public CodexSpace Space()
         {
-            var space = OnCodex<Space>(api => api.SpaceAsync());
+            var space = OnCodex(api => api.SpaceAsync());
             return mapper.Map(space);
         }
 
@@ -189,7 +223,7 @@ namespace CodexPlugin
 
         private Address GetAddress()
         {
-            return Container.Containers.Single().GetAddress(log, CodexContainerRecipe.ApiPortTag);
+            return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
         }
 
         private string GetHttpId()
@@ -255,4 +289,18 @@ namespace CodexPlugin
             log.Log($"{GetName()} {msg}");
         }
     }
+
+    public class UploadInput
+    {
+        public UploadInput(string contentType, string contentDisposition, FileStream fileStream)
+        {
+            ContentType = contentType;
+            ContentDisposition = contentDisposition;
+            FileStream = fileStream;
+        }
+
+        public string ContentType { get; }
+        public string ContentDisposition { get; }
+        public FileStream FileStream { get; }
+    }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,8 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-64b82de-dist-tests";
-
+        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";
@@ -110,7 +109,7 @@ namespace CodexPlugin
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
                 var account = marketplaceSetup.EthAccountSetup.GetNew();
-                AddEnvVar("PRIV_KEY", account.PrivateKey);
+                AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
                 Additional(account);
 
                 SetCommandOverride(marketplaceSetup);

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -14,12 +14,16 @@ namespace CodexPlugin
     {
         string GetName();
         string GetPeerId();
-        DebugInfo GetDebugInfo();
+        DebugInfo GetDebugInfo(bool log = false);
+        string GetSpr();
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
         ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
+        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
+        LocalDataset DownloadStreamless(ContentId cid);
+        LocalDataset DownloadManifestOnly(ContentId cid);
         LocalDatasetList LocalFiles();
         CodexSpace Space();
         void ConnectToPeer(ICodexNode node);
@@ -119,14 +123,22 @@ namespace CodexPlugin
             return peerId;
         }
 
-        public DebugInfo GetDebugInfo()
+        public DebugInfo GetDebugInfo(bool log = false)
         {
             var debugInfo = CodexAccess.GetDebugInfo();
-            var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
-            Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
+            if (log)
+            {
+                var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
+                Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
+            }
             return debugInfo;
         }
 
+        public string GetSpr()
+        {
+            return CodexAccess.GetSpr();
+        }
+
         public DebugPeer GetDebugPeer(string peerId)
         {
             return CodexAccess.GetDebugPeer(peerId);
@@ -138,6 +150,11 @@ namespace CodexPlugin
         }
 
         public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
+        {
+            return UploadFile(file, "application/x-binary", $"attachment; filename=\"{file.Filename}\"", onFailure);
+        }
+
+        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenRead(file.Filename);
             var uniqueId = Guid.NewGuid().ToString();
@@ -145,10 +162,11 @@ namespace CodexPlugin
 
             hooks.OnFileUploading(uniqueId, size);
 
-            var logMessage = $"Uploading file {file.Describe()}...";
+            var input = new UploadInput(contentType, contentDisposition, fileStream);
+            var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
             var measurement = Stopwatch.Measure(log, logMessage, () =>
             {
-                return CodexAccess.UploadFile(fileStream, onFailure);
+                return CodexAccess.UploadFile(input, onFailure);
             });
 
             var response = measurement.Value;
@@ -185,6 +203,16 @@ namespace CodexPlugin
             return file;
         }
 
+        public LocalDataset DownloadStreamless(ContentId cid)
+        {
+            return CodexAccess.DownloadStreamless(cid);
+        }
+
+        public LocalDataset DownloadManifestOnly(ContentId cid)
+        {
+            return CodexAccess.DownloadManifestOnly(cid);
+        }
+
         public LocalDatasetList LocalFiles()
         {
             return CodexAccess.LocalFiles();
@@ -244,6 +272,11 @@ namespace CodexPlugin
             Version = debugInfo.Version;
         }
 
+        public override string ToString()
+        {
+            return $"CodexNode:{GetName()}";
+        }
+
         private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)
         {
             // The peer we want to connect is in a different pod.
@@ -259,10 +292,27 @@ namespace CodexPlugin
         private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
         {
             using var fileStream = File.OpenWrite(file.Filename);
+            var timeout = tools.TimeSet.HttpCallTimeout();
             try
             {
-                using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
-                downloadStream.CopyTo(fileStream);
+                // Type of stream generated by openAPI client does not support timeouts.
+                var start = DateTime.UtcNow;
+                var cts = new CancellationTokenSource();
+                var downloadTask = Task.Run(() =>
+                {
+                    using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
+                    downloadStream.CopyTo(fileStream);
+                }, cts.Token);
+                
+                while (DateTime.UtcNow - start < timeout)
+                {
+                    if (downloadTask.IsFaulted) throw downloadTask.Exception;
+                    if (downloadTask.IsCompletedSuccessfully) return;
+                    Thread.Sleep(100);
+                }
+
+                cts.Cancel();
+                throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
             }
             catch
             {

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -20,7 +20,7 @@ namespace CodexPlugin
 
         public void Announce()
         {
-            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
+            Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
         }
 
         public void AddMetadata(IAddMetadata metadata)
@@ -55,6 +55,10 @@ namespace CodexPlugin
             {
                 mconfig.GethNode.SendEth(node, mconfig.MarketplaceSetup.InitialEth);
                 mconfig.CodexContracts.MintTestTokens(node, mconfig.MarketplaceSetup.InitialTestTokens);
+
+                Log($"Send {mconfig.MarketplaceSetup.InitialEth} and " +
+                    $"minted {mconfig.MarketplaceSetup.InitialTestTokens} for " +
+                    $"{node.GetName()} (address: {node.EthAddress})");
             }
         }
 
@@ -70,5 +74,10 @@ namespace CodexPlugin
             setup(codexSetup);
             return codexSetup;
         }
+
+        private void Log(string msg)
+        {
+            tools.GetLog().Log(msg);
+        }
     }
 }

ProjectPlugins/CodexPlugin/CodexPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -16,8 +16,8 @@ namespace CodexPlugin
                 Spr = debugInfo.Spr,
                 Addrs = debugInfo.Addrs.ToArray(),
                 AnnounceAddresses = JArray(debugInfo.AdditionalProperties, "announceAddresses").Select(x => x.ToString()).ToArray(),
-                Version = MapDebugInfoVersion(JObject(debugInfo.AdditionalProperties, "codex")),
-                Table = MapDebugInfoTable(JObject(debugInfo.AdditionalProperties, "table"))
+                Version = Map(debugInfo.Codex),
+                Table = Map(debugInfo.Table)
             };
         }
 
@@ -63,6 +63,26 @@ namespace CodexPlugin
             };
         }
 
+        public StorageAvailability[] Map(ICollection<SalesAvailabilityREAD> availabilities)
+        {
+            return availabilities.Select(a => Map(a)).ToArray();
+        }
+
+        public StorageAvailability Map(SalesAvailabilityREAD availability)
+        {
+            return new StorageAvailability
+            (
+                ToByteSize(availability.TotalSize),
+                ToTimespan(availability.Duration),
+                new TestToken(ToBigIng(availability.MinPrice)),
+                new TestToken(ToBigIng(availability.MaxCollateral))
+            )
+            {
+                Id = availability.Id,
+                FreeSpace = ToByteSize(availability.FreeSize),
+            };
+        }
+
         // TODO: Fix openapi spec for this call.
         //public StoragePurchase Map(CodexOpenApi.Purchase purchase)
         //{
@@ -105,19 +125,6 @@ namespace CodexPlugin
         //    };
         //}
 
-        public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD read)
-        {
-            return new StorageAvailability(
-                totalSpace: new ByteSize(Convert.ToInt64(read.TotalSize)),
-                maxDuration: TimeSpan.FromSeconds(Convert.ToDouble(read.Duration)),
-                minPriceForTotalSpace: new TestToken(BigInteger.Parse(read.MinPrice)),
-                maxCollateral: new TestToken(BigInteger.Parse(read.MaxCollateral))
-            )
-            {
-                Id = read.Id
-            };
-        }
-
         public CodexSpace Map(Space space)
         {
             return new CodexSpace
@@ -129,47 +136,45 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoVersion MapDebugInfoVersion(JObject obj)
+        private DebugInfoVersion Map(CodexVersion obj)
         {
             return new DebugInfoVersion
             {
-                Version = StringOrEmpty(obj, "version"),
-                Revision = StringOrEmpty(obj, "revision")
+                Version = obj.Version,
+                Revision = obj.Revision
             };
         }
 
-        private DebugInfoTable MapDebugInfoTable(JObject obj)
+        private DebugInfoTable Map(PeersTable obj)
         {
             return new DebugInfoTable
             {
-                LocalNode = MapDebugInfoTableNode(obj.GetValue("localNode")),
-                Nodes = MapDebugInfoTableNodeArray(obj.GetValue("nodes") as JArray)
+                LocalNode = Map(obj.LocalNode),
+                Nodes = Map(obj.Nodes)
             };
         }
 
-        private DebugInfoTableNode MapDebugInfoTableNode(JToken? token)
+        private DebugInfoTableNode Map(Node? token)
         {
-            var obj = token as JObject;
-            if (obj == null) return new DebugInfoTableNode();
-
+            if (token == null) return new DebugInfoTableNode();
             return new DebugInfoTableNode
             {
-                Address = StringOrEmpty(obj, "address"),
-                NodeId = StringOrEmpty(obj, "nodeId"),
-                PeerId = StringOrEmpty(obj, "peerId"),
-                Record = StringOrEmpty(obj, "record"),
-                Seen = Bool(obj, "seen")
+                Address = token.Address,
+                NodeId = token.NodeId,
+                PeerId = token.PeerId,
+                Record = token.Record,
+                Seen = token.Seen
             };
         }
 
-        private DebugInfoTableNode[] MapDebugInfoTableNodeArray(JArray? nodes)
+        private DebugInfoTableNode[] Map(ICollection<Node> nodes)
         {
             if (nodes == null || nodes.Count == 0)
             {
                 return new DebugInfoTableNode[0];
             }
 
-            return nodes.Select(MapDebugInfoTableNode).ToArray();
+            return nodes.Select(Map).ToArray();
         }
 
         private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)
@@ -177,8 +182,8 @@ namespace CodexPlugin
             return new Manifest
             {
                 BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
-                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.OriginalBytes)),
-                RootHash = manifest.RootHash,
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
+                RootHash = manifest.TreeCid,
                 Protected = manifest.Protected
             };
         }
@@ -222,5 +227,20 @@ namespace CodexPlugin
         {
             return t.TstWei.ToString("D");
         }
+
+        private BigInteger ToBigIng(string tokens)
+        {
+            return BigInteger.Parse(tokens);
+        }
+
+        private TimeSpan ToTimespan(string duration)
+        {
+            return TimeSpan.FromSeconds(Convert.ToInt32(duration));
+        }
+
+        private ByteSize ToByteSize(string size)
+        {
+            return new ByteSize(Convert.ToInt64(size));
+        }
     }
 }

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -7,6 +7,7 @@ namespace CodexPlugin
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(StorageAvailability availability);
+        StorageAvailability[] GetAvailabilities();
         IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
     }
 
@@ -61,6 +62,14 @@ namespace CodexPlugin
             return response.Id;
         }
 
+        public StorageAvailability[] GetAvailabilities()
+        {
+            var result = codexAccess.GetAvailabilities();
+            Log($"Got {result.Length} availabilities:");
+            foreach (var a in result) a.Log(log);
+            return result;
+        }
+
         private void Log(string msg)
         {
             log.Log($"{codexAccess.Container.Containers.Single().Name} {msg}");
@@ -81,6 +90,12 @@ namespace CodexPlugin
             throw new NotImplementedException();
         }
 
+        public StorageAvailability[] GetAvailabilities()
+        {
+            Unavailable();
+            throw new NotImplementedException();
+        }
+
         private void Unavailable()
         {
             FrameworkAssert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it.");

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -1,7 +1,5 @@
 using CodexContractsPlugin;
-using CodexOpenApi;
 using Logging;
-using System.Data;
 using Utils;
 
 namespace CodexPlugin
@@ -84,10 +82,11 @@ namespace CodexPlugin
         public TimeSpan MaxDuration { get; }
         public TestToken MinPriceForTotalSpace { get; }
         public TestToken MaxCollateral { get; } 
+        public ByteSize FreeSpace { get; set; } = ByteSize.Zero;
 
         public void Log(ILog log)
         {
-            log.Log($"Making storage available... (" +
+            log.Log($"Storage Availability: (" +
                 $"totalSize: {TotalSpace}, " +
                 $"maxDuration: {Time.FormatDuration(MaxDuration)}, " + 
                 $"minPriceForTotalSpace: {MinPriceForTotalSpace}, " +

ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs

@@ -1,4 +1,6 @@
-using CodexPlugin.Hooks;
+using CodexContractsPlugin;
+using CodexPlugin.Hooks;
+using GethPlugin;
 using Logging;
 using Newtonsoft.Json;
 using Utils;
@@ -12,7 +14,7 @@ namespace CodexPlugin
         ContentId ContentId { get; }
         void WaitForStorageContractSubmitted();
         void WaitForStorageContractStarted();
-        void WaitForStorageContractFinished();
+        void WaitForStorageContractFinished(ICodexContracts contracts);
     }
 
     public class StoragePurchaseContract : IStoragePurchaseContract
@@ -62,7 +64,7 @@ namespace CodexPlugin
             AssertDuration(SubmittedToStarted, timeout, nameof(SubmittedToStarted));
         }
 
-        public void WaitForStorageContractFinished()
+        public void WaitForStorageContractFinished(ICodexContracts contracts)
         {
             if (!contractStartedUtc.HasValue)
             {
@@ -74,6 +76,13 @@ namespace CodexPlugin
             contractFinishedUtc = DateTime.UtcNow;
             LogFinishedDuration();
             AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
+
+            contracts.WaitUntilNextPeriod();
+            contracts.WaitUntilNextPeriod();
+
+            var blocks = 3;
+            Log($"Waiting {blocks} blocks for nodes to process payouts...");
+            Thread.Sleep(GethContainerRecipe.BlockInterval * blocks);
         }
 
         public StoragePurchase GetPurchaseStatus(string purchaseId)

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -23,6 +23,8 @@ components:
     Id:
       type: string
       description: 32bits identifier encoded in hex-decimal string.
+      minLength: 66
+      maxLength: 66
       example: 0x...
 
     BigInt:
@@ -81,33 +83,46 @@ components:
         id:
           $ref: "#/components/schemas/PeerId"
 
-    ErasureParameters:
-      type: object
-      properties:
-        totalChunks:
-          type: integer
-
-    PoRParameters:
-      description: Parameters for Proof of Retrievability
-      type: object
-      properties:
-        u:
-          type: string
-        publicKey:
-          type: string
-        name:
-          type: string
-
     Content:
       type: object
       description: Parameters specifying the content
       properties:
         cid:
           $ref: "#/components/schemas/Cid"
-        erasure:
-          $ref: "#/components/schemas/ErasureParameters"
-        por:
-          $ref: "#/components/schemas/PoRParameters"
+
+    Node:
+      type: object
+      properties:
+        nodeId:
+          type: string
+        peerId:
+          type: string
+        record:
+          type: string
+        address:
+          type: string
+        seen:
+          type: boolean
+
+    CodexVersion:
+      type: object
+      properties:
+        version:
+          type: string
+          example: v0.1.7
+        revision:
+          type: string
+          example: 0c647d8
+
+    PeersTable:
+      type: object
+      properties:
+        localNode:
+          $ref: "#/components/schemas/Node"
+        nodes:
+          type: array
+          items:
+            $ref: "#/components/schemas/Node"
 
     DebugInfo:
       type: object
@@ -123,6 +138,10 @@ components:
           description: Path of the data repository where all nodes data are stored
         spr:
           $ref: "#/components/schemas/SPR"
+        table:
+          $ref: "#/components/schemas/PeersTable"
+        codex:
+          $ref: "#/components/schemas/CodexVersion"
 
     SalesAvailability:
       type: object
@@ -136,7 +155,7 @@ components:
           $ref: "#/components/schemas/Duration"
         minPrice:
           type: string
-          description: Minimum price to be paid (in amount of tokens) as decimal string
+          description: Minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration as decimal string
         maxCollateral:
           type: string
           description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
@@ -168,7 +187,39 @@ components:
           $ref: "#/components/schemas/StorageRequest"
         slotIndex:
           type: string
-          description: Slot Index as hexadecimal string
+          description: Slot Index as decimal string
+
+    SlotAgent:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/SlotId"
+        slotIndex:
+          type: string
+          description: Slot Index as decimal string
+        requestId:
+          $ref: "#/components/schemas/Id"
+        request:
+          $ref: "#/components/schemas/StorageRequest"
+        reservation:
+          $ref: "#/components/schemas/Reservation"
+        state:
+          type: string
+          description: Description of the slot's
+          enum:
+            - SaleCancelled
+            - SaleDownloading
+            - SaleErrored
+            - SaleFailed
+            - SaleFilled
+            - SaleFilling
+            - SaleFinished
+            - SaleIgnored
+            - SaleInitialProving
+            - SalePayout
+            - SalePreparing
+            - SaleProving
+            - SaleUnknown
 
     Reservation:
       type: object
@@ -183,7 +234,7 @@ components:
           $ref: "#/components/schemas/Id"
         slotIndex:
           type: string
-          description: Slot Index as hexadecimal string
+          description: Slot Index as decimal string
 
     StorageRequestCreation:
       type: object
@@ -259,6 +310,15 @@ components:
         state:
           type: string
           description: Description of the Request's state
+          enum:
+            - cancelled
+            - error
+            - failed
+            - finished
+            - pending
+            - started
+            - submitted
+            - unknown
         error:
           type: string
           description: If Request failed, then here is presented the error message
@@ -284,10 +344,10 @@ components:
     ManifestItem:
       type: object
       properties:
-        rootHash:
+        treeCid:
           $ref: "#/components/schemas/Cid"
-          description: "Root hash of the content"
-        originalBytes:
+          description: "Unique data identifier"
+        datasetSize:
           type: integer
           format: int64
           description: "Length of original content in bytes"
@@ -297,6 +357,22 @@ components:
         protected:
           type: boolean
           description: "Indicates if content is protected by erasure-coding"
+        filename:
+          type: string
+          nullable: true
+          description: "The original name of the uploaded content (optional)"
+          example: codex.png
+        mimetype:
+          type: string
+          nullable: true
+          description: "The original mimetype of the uploaded content (optional)"
+          example: image/png
+        uploadedAt:
+          type: integer
+          format: int64
+          nullable: true
+          description: "The UTC upload timestamp in seconds"
+          example: 1729244192
 
     Space:
       type: object
@@ -308,15 +384,15 @@ components:
         quotaMaxBytes:
           type: integer
           format: int64
-          description: "Maximum storage space used by the node"
+          description: "Maximum storage space (in bytes) available for the node in Codex's local repository."
         quotaUsedBytes:
           type: integer
           format: int64
-          description: "Amount of storage space currently in use"
+          description: "Amount of storage space (in bytes) currently used for storing files in Codex's local repository."
         quotaReservedBytes:
           type: integer
           format: int64
-          description: "Amount of storage space reserved"
+          description: "Amount of storage reserved (in bytes) in the Codex's local repository for future use when storage requests will be picked up and hosted by the node using node's availabilities. This does not include the storage currently in use."
 
 servers:
   - url: "http://localhost:8080/api/codex/v1"
@@ -382,12 +458,29 @@ paths:
           description: Invalid CID is specified
         "404":
           description: Content specified by the CID is not found
+        "422":
+          description: The content type is not a valid content type or the filename is not valid
         "500":
           description: Well it was bad-bad
     post:
       summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
       tags: [ Data ]
       operationId: upload
+      parameters:
+        - name: content-type
+          in: header
+          required: false
+          description: The content type of the file. Must be valid.
+          schema:
+            type: string
+            example: "image/png"
+        - name: content-disposition
+          in: header
+          required: false
+          description: The content disposition used to send the filename.
+          schema:
+            type: string
+            example: "attachment; filename=\"codex.png\""
       requestBody:
         content:
           application/octet-stream:
@@ -433,10 +526,36 @@ paths:
           description: Well it was bad-bad
 
   "/data/{cid}/network":
+    post:
+      summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
+      tags: [ Data ]
+      operationId: downloadNetwork
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File to be downloaded."
+      responses:
+        "200":
+          description: Manifest information for download that has been started.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
+  "/data/{cid}/network/stream":
     get:
       summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
       tags: [ Data ]
-      operationId: downloadNetwork
+      operationId: downloadNetworkStream
       parameters:
         - in: path
           name: cid
@@ -459,6 +578,32 @@ paths:
         "500":
           description: Well it was bad-bad
 
+  "/data/{cid}/network/manifest":
+    get:
+      summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
+      tags: [ Data ]
+      operationId: downloadNetworkManifest
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File for which the manifest is to be downloaded."
+      responses:
+        "200":
+          description: Manifest information.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataItem"
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Failed to download dataset manifest
+        "500":
+          description: Well it was bad-bad
+
   "/space":
     get:
       summary: "Gets a summary of the storage space allocation of the node."
@@ -491,7 +636,7 @@ paths:
                   $ref: "#/components/schemas/Slot"
 
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/sales/slots/{slotId}":
     get:
@@ -511,7 +656,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/Slot"
+                $ref: "#/components/schemas/SlotAgent"
 
         "400":
           description: Invalid or missing SlotId
@@ -520,13 +665,13 @@ paths:
           description: Host is not in an active sale for the slot
 
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/sales/availability":
     get:
       summary: "Returns storage that is for sale"
       tags: [ Marketplace ]
-      operationId: getOfferedStorage
+      operationId: getAvailabilities
       responses:
         "200":
           description: Retrieved storage availabilities of the node
@@ -535,11 +680,11 @@ paths:
               schema:
                 type: array
                 items:
-                  $ref: "#/components/schemas/SalesAvailability"
+                  $ref: "#/components/schemas/SalesAvailabilityREAD"
         "500":
           description: Error getting unused availabilities
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
     post:
       summary: "Offers storage for sale"
@@ -564,7 +709,7 @@ paths:
         "500":
           description: Error reserving availability
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
   "/sales/availability/{id}":
     patch:
       summary: "Updates availability"
@@ -597,10 +742,10 @@ paths:
         "500":
           description: Error reserving availability
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/sales/availability/{id}/reservations":
-    patch:
+    get:
       summary: "Get availability's reservations"
       description: Return's list of Reservations for ongoing Storage Requests that the node hosts.
       operationId: getReservations
@@ -628,7 +773,7 @@ paths:
         "500":
           description: Error getting reservations
         "503":
-          description: Sales are unavailable
+          description: Persistence is not enabled
 
   "/storage/request/{cid}":
     post:
@@ -659,7 +804,7 @@ paths:
         "404":
           description: Request ID not found
         "503":
-          description: Purchasing is unavailable
+          description: Persistence is not enabled
 
   "/storage/purchases":
     get:
@@ -676,7 +821,7 @@ paths:
                 items:
                   type: string
         "503":
-          description: Purchasing is unavailable
+          description: Persistence is not enabled
 
   "/storage/purchases/{id}":
     get:
@@ -702,9 +847,9 @@ paths:
         "404":
           description: Purchase not found
         "503":
-          description: Purchasing is unavailable
+          description: Persistence is not enabled
 
-  "/node/spr":
+  "/spr":
     get:
       summary: "Get Node's SPR"
       operationId: getSPR
@@ -722,7 +867,7 @@ paths:
         "503":
           description: Node SPR not ready, try again later
 
-  "/node/peerid":
+  "/peerid":
     get:
       summary: "Get Node's PeerID"
       operationId: getPeerId
@@ -770,4 +915,4 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/DebugInfo"
+                $ref: "#/components/schemas/DebugInfo"

ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj

@@ -2,9 +2,13 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+  </ItemGroup>
+
 </Project>

ProjectPlugins/CodexPluginPrebuild/Program.cs

@@ -1,5 +1,6 @@
 using System.Security.Cryptography;
 using System.Text;
+using Utils;
 
 public static class Program
 {
@@ -40,32 +41,9 @@ public static class Program
 
     private static string FindCodexPluginFolder()
     {
-        var current = Directory.GetCurrentDirectory();
-
-        while (true)
-        {
-            var localFolders = Directory.GetDirectories(current);
-            var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
-            if (projectPluginsFolders.Length == 1)
-            {
-                return Path.Combine(projectPluginsFolders.Single(), CodexPluginFolderName);
-            }
-            var codexPluginFolders = localFolders.Where(l => l.EndsWith(CodexPluginFolderName)).ToArray();
-            if (codexPluginFolders.Length == 1)
-            {
-                return codexPluginFolders.Single();
-            }
-
-            var parent = Directory.GetParent(current);
-            if (parent == null)
-            {
-                var msg = $"Unable to locate '{CodexPluginFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
-                Console.WriteLine(msg);
-                throw new Exception(msg);
-            }
-
-            current = parent.FullName;
-        }
+        var folder = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin");
+        if (!Directory.Exists(folder)) throw new Exception("CodexPlugin folder not found. Expected: " + folder);
+        return folder;
     }
 
     private static string CreateHash(string openApiFile)

ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/GethPlugin/EthTokenExtensions.cs

@@ -28,7 +28,13 @@
 
         public override string ToString()
         {
-            return $"{Eth} Eth";
+            var weiOnly = Wei % TokensIntExtensions.WeiPerEth;
+
+            var tokens = new List<string>();
+            if (Eth > 0) tokens.Add($"{Eth} Eth");
+            if (weiOnly > 0) tokens.Add($"{weiOnly} Wei");
+
+            return string.Join(" + ", tokens);
         }
     }
 

ProjectPlugins/GethPlugin/GethContainerRecipe.cs

@@ -6,6 +6,7 @@ namespace GethPlugin
     public class GethContainerRecipe : ContainerRecipeFactory
     {
         public static string DockerImage { get; } = "codexstorage/dist-tests-geth:latest";
+        public static TimeSpan BlockInterval { get; } = TimeSpan.FromSeconds(1.0);
         private const string defaultArgs = "--ipcdisable --syncmode full";
 
         public const string HttpPortTag = "http_port";

ProjectPlugins/GethPlugin/GethNode.cs

@@ -57,7 +57,7 @@ namespace GethPlugin
 
         protected override NethereumInteraction StartInteraction()
         {
-            var address = StartResult.Container.GetAddress(log, GethContainerRecipe.HttpPortTag);
+            var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
             var account = StartResult.Account;
 
             var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);

ProjectPlugins/GethPlugin/GethPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -6,14 +6,14 @@ namespace MetricsPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray(), scrapeInterval);
         }
 
-        public static RunningPod DeployMetricsCollector(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            return Plugin(ci).DeployMetricsCollector(scrapeTargets);
+            return Plugin(ci).DeployMetricsCollector(scrapeTargets, scrapeInterval);
         }
 
         public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget)
@@ -26,19 +26,19 @@ namespace MetricsPlugin
             return Plugin(ci).WrapMetricsCollectorDeployment(metricsPod, scrapeTarget);
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasManyMetricScrapeTargets[] manyScrapeTargets)
         {
-            return ci.GetMetricsFor(manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IHasMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets)
         {
-            return ci.GetMetricsFor(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
+            return ci.GetMetricsFor(scrapeInterval, scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray());
         }
 
-        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, params IMetricsScrapeTarget[] scrapeTargets)
+        public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IMetricsScrapeTarget[] scrapeTargets)
         {
-            var rc = ci.DeployMetricsCollector(scrapeTargets);
+            var rc = ci.DeployMetricsCollector(scrapeInterval, scrapeTargets);
             return scrapeTargets.Select(t => ci.WrapMetricsCollector(rc, t)).ToArray();
         }
 

ProjectPlugins/MetricsPlugin/MetricsAccess.cs

@@ -7,7 +7,7 @@ namespace MetricsPlugin
     public interface IMetricsAccess : IHasContainer
     {
         string TargetName { get; }
-        Metrics? GetAllMetrics();
+        Metrics GetAllMetrics();
         MetricsSet GetMetric(string metricName);
         MetricsSet GetMetric(string metricName, TimeSpan timeout);
     }
@@ -27,7 +27,7 @@ namespace MetricsPlugin
         public string TargetName { get; }
         public RunningContainer Container => query.RunningContainer;
 
-        public Metrics? GetAllMetrics()
+        public Metrics GetAllMetrics()
         {
             return query.GetAllMetricsForNode(target);
         }
@@ -54,11 +54,10 @@ namespace MetricsPlugin
             }
         }
 
-        private MetricsSet? GetMostRecent(string metricName)
+        private MetricsSet GetMostRecent(string metricName)
         {
             var result = query.GetMostRecent(metricName, target);
-            if (result == null) return null;
-            return result.Sets.LastOrDefault();
+            return result.Sets.Last();
         }
     }
 }

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -31,9 +31,9 @@ namespace MetricsPlugin
         {
         }
 
-        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets)
+        public RunningPod DeployMetricsCollector(IMetricsScrapeTarget[] scrapeTargets, TimeSpan scrapeInterval)
         {
-            return starter.CollectMetricsFor(scrapeTargets);
+            return starter.CollectMetricsFor(scrapeTargets, scrapeInterval);
         }
 
         public IMetricsAccess WrapMetricsCollectorDeployment(RunningPod runningPod, IMetricsScrapeTarget target)

ProjectPlugins/MetricsPlugin/MetricsPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -15,7 +15,7 @@ namespace MetricsPlugin
         {
             RunningContainer = runningContainer;
             log = tools.GetLog();
-            var address = RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag);
+            var address = RunningContainer.GetAddress(PrometheusContainerRecipe.PortTag);
             endpoint = tools
                 .CreateHttp(address.ToString())
                 .CreateEndpoint(address, "/api/v1/");
@@ -23,10 +23,10 @@ namespace MetricsPlugin
 
         public RunningContainer RunningContainer { get; }
 
-        public Metrics? GetMostRecent(string metricName, IMetricsScrapeTarget target)
+        public Metrics GetMostRecent(string metricName, IMetricsScrapeTarget target)
         {
             var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
-            if (response == null) return null;
+            if (response == null) throw new Exception($"Failed to get most recent metric: {metricName}");
 
             var result = new Metrics
             {
@@ -44,19 +44,20 @@ namespace MetricsPlugin
             return result;
         }
 
-        public Metrics? GetMetrics(string metricName)
+        public Metrics GetMetrics(string metricName)
         {
             var response = GetAll(metricName);
-            if (response == null) return null;
+            if (response == null) throw new Exception($"Failed to get metrics by name: {metricName}");
             var result = MapResponseToMetrics(response);
             Log(metricName, result);
             return result;
         }
 
-        public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
+        public Metrics GetAllMetricsForNode(IMetricsScrapeTarget target)
         {
-            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
-            if (response.status != "success") return null;
+            var instanceString = GetInstanceStringForNode(target);
+            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={instanceString}{GetQueryTimeRange()}");
+            if (response.status != "success") throw new Exception($"Failed to get metrics for target: {instanceString}");
             var result = MapResponseToMetrics(response);
             Log(target, result);
             return result;
@@ -80,18 +81,32 @@ namespace MetricsPlugin
         {
             return new Metrics
             {
-                Sets = response.data.result.Select(r =>
-                {
-                    return new MetricsSet
-                    {
-                        Name = r.metric.__name__,
-                        Instance = r.metric.instance,
-                        Values = MapMultipleValues(r.values)
-                    };
-                }).ToArray()
+                Sets = response.data.result.Select(CreateMetricsSet).ToArray()
             };
         }
 
+        private MetricsSet CreateMetricsSet(PrometheusQueryResponseDataResultEntry r)
+        {
+            var result = new MetricsSet
+            {
+                Name = r.metric.__name__,
+                Instance = r.metric.instance,
+                Values = MapMultipleValues(r.values)
+            };
+
+            if (!string.IsNullOrEmpty(r.metric.file) && !string.IsNullOrEmpty(r.metric.line) && !string.IsNullOrEmpty(r.metric.proc))
+            {
+                result.AsyncProfiler = new AsyncProfilerMetrics
+                {
+                    File = r.metric.file,
+                    Line = r.metric.line,
+                    Proc = r.metric.proc
+                };
+            }
+
+            return result;
+        }
+
         private MetricsSetValue[] MapSingleValue(object[] value)
         {
             if (value != null && value.Length > 0)
@@ -220,14 +235,28 @@ namespace MetricsPlugin
     {
         public string Name { get; set; } = string.Empty;
         public string Instance { get; set; } = string.Empty;
+        public AsyncProfilerMetrics? AsyncProfiler { get; set; } = null;
         public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
 
         public override string ToString()
         {
-            return $"{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
+            var prefix = "";
+            if (AsyncProfiler != null)
+            {
+                prefix = $"proc: '{AsyncProfiler.Proc}' in '{AsyncProfiler.File}:{AsyncProfiler.Line}'";
+            }
+
+            return $"{prefix}{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
         }
     }
 
+    public class AsyncProfilerMetrics
+    {
+        public string File { get; set; } = string.Empty;
+        public string Line { get; set; } = string.Empty;
+        public string Proc { get; set; } = string.Empty;
+    }
+
     public class MetricsSetValue
     {
         public DateTime Timestamp { get; set; }
@@ -263,6 +292,10 @@ namespace MetricsPlugin
         public string __name__ { get; set; } = string.Empty;
         public string instance { get; set; } = string.Empty;
         public string job { get; set; } = string.Empty;
+        // Async profiler output.
+        public string? file { get; set; } = null;
+        public string? line { get; set; } = null;
+        public string? proc { get; set; } = null;
     }
 
     public class PrometheusAllNamesResponse

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -16,13 +16,13 @@ namespace MetricsPlugin
             this.tools = tools;
         }
 
-        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets)
+        public RunningPod CollectMetricsFor(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
         {
             if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
 
             Log($"Starting metrics server for {targets.Length} targets...");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets, scrapeInterval)));
 
             var workflow = tools.CreateWorkflow();
             var runningContainers = workflow.Start(1, recipe, startupConfig).WaitForOnline();
@@ -48,12 +48,16 @@ namespace MetricsPlugin
             tools.GetLog().Log(msg);
         }
 
-        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets)
+        private string GeneratePrometheusConfig(IMetricsScrapeTarget[] targets, TimeSpan scrapeInterval)
         {
+            var secs = Convert.ToInt32(scrapeInterval.TotalSeconds);
+            if (secs < 1) throw new Exception("ScrapeInterval can't be < 1s");
+            if (secs > 60) throw new Exception("ScrapeInterval can't be > 60s");
+
             var config = "";
             config += "global:\n";
-            config += "  scrape_interval: 10s\n";
-            config += "  scrape_timeout: 10s\n";
+            config += $"  scrape_interval: {secs}s\n";
+            config += $"  scrape_timeout: {secs}s\n";
             config += "\n";
             config += "scrape_configs:\n";
             config += "  - job_name: services\n";
@@ -80,7 +84,7 @@ namespace MetricsPlugin
     {
         public static string FormatTarget(ILog log, IMetricsScrapeTarget target)
         {
-            var a = target.Container.GetAddress(log, target.MetricsPortTag);
+            var a = target.Container.GetAddress(target.MetricsPortTag);
             var host = a.Host.Replace("http://", "").Replace("https://", "");
             return $"{host}:{a.Port}";
         }

README.md

@@ -2,7 +2,7 @@
 
 This project allows you to write tools and tests that control and interact with container-based applications to form a distributed system in a controlled, reproducible environment.
 
-Dotnet: v7.0  
+Dotnet: v8.0  
 Kubernetes: v1.25.4  
 Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp  
 Nethereum: v4.14.0

Tests/CodexContinuousTests/CodexContinuousTests.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
@@ -14,8 +14,8 @@
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
-    <ProjectReference Include="..\CodexTests\CodexTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\ExperimentalTests\ExperimentalTests.csproj" />
   </ItemGroup>
 
 </Project>

Tests/CodexContinuousTests/StartupChecker.cs

@@ -98,7 +98,7 @@ namespace ContinuousTests
             {
                 cancelToken.ThrowIfCancellationRequested();
 
-                var address = n.Container.GetAddress(log, CodexContainerRecipe.ApiPortTag);
+                var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
                 log.Log($"Checking {n.Container.Name} @ '{address}'...");
 
                 if (EnsureOnline(log, n))

Tests/CodexLongTests/CodexTestsLong.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
@@ -14,8 +14,8 @@
 
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
-    <ProjectReference Include="..\CodexTests\CodexTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\ExperimentalTests\ExperimentalTests.csproj" />
   </ItemGroup>
 
 </Project>

Tests/CodexReleaseTests/CodexReleaseTests.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="NUnit" Version="3.13.3" />
+	<PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
+	<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ExperimentalTests\ExperimentalTests.csproj" />
+  </ItemGroup>
+
+</Project>

Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs

@@ -0,0 +1,47 @@
+using CodexPlugin;
+using CodexTests;
+using FileUtils;
+using NUnit.Framework;
+using System.Diagnostics;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    public class InterruptUploadTest : CodexDistTest
+    {
+        [Test]
+        public void UploadInterruptTest()
+        {
+            var nodes = StartCodex(10);
+
+            var tasks = nodes.Select(n => Task<bool>.Run(() => RunInterruptUploadTest(n)));
+            Task.WaitAll(tasks.ToArray());
+
+            Assert.That(tasks.Select(t => t.Result).All(r => r == true));
+
+            WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), nodes);
+        }
+
+        private bool RunInterruptUploadTest(ICodexNode node)
+        {
+            var file = GenerateTestFile(300.MB());
+
+            var process = StartCurlUploadProcess(node, file);
+
+            Thread.Sleep(500);
+            process.Kill();
+            Thread.Sleep(1000);
+
+            var log = Ci.DownloadLog(node);
+            return !log.GetLinesContaining("Unhandled exception in async proc, aborting").Any();
+        }
+
+        private Process StartCurlUploadProcess(ICodexNode node, TrackedFile file)
+        {
+            var apiAddress = node.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
+            var codexUrl = $"{apiAddress}/api/codex/v1/data";
+            var filePath = file.Filename;
+            return Process.Start("curl", $"-X POST {codexUrl} -H \"Content-Type: application/octet-stream\" -T {filePath}");
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/ManifestOnlyDownloadTest.cs

@@ -0,0 +1,33 @@
+using CodexTests;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class ManifestOnlyDownloadTest : CodexDistTest
+    {
+        [Test]
+        public void ManifestOnlyTest()
+        {
+            var uploader = StartCodex();
+            var downloader = StartCodex(s => s.WithBootstrapNode(uploader));
+
+            var file = GenerateTestFile(2.GB());
+            var size = file.GetFilesize().SizeInBytes;
+            var cid = uploader.UploadFile(file);
+
+            var startSpace = downloader.Space();
+            var localDataset = downloader.DownloadManifestOnly(cid);
+
+            Thread.Sleep(1000);
+
+            var spaceDiff = startSpace.FreeBytes - downloader.Space().FreeBytes;
+
+            Assert.That(spaceDiff, Is.LessThan(64.KB().SizeInBytes));
+
+            Assert.That(localDataset.Cid, Is.EqualTo(cid));
+            Assert.That(localDataset.Manifest.OriginalBytes.SizeInBytes, Is.EqualTo(file.GetFilesize().SizeInBytes));
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/OneClientTest.cs

@@ -1,20 +1,26 @@
 using CodexPlugin;
+using CodexTests;
 using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
 using Utils;
 
-namespace CodexTests.BasicTests
+namespace CodexReleaseTests.DataTests
 {
     [TestFixture]
-    public class OneClientTests : CodexDistTest
+    public class OneClientTest : CodexDistTest
     {
         [Test]
-        public void OneClientTest()
+        public void OneClient()
         {
-            var primary = StartCodex();
+            var node = StartCodex();
 
-            PerformOneClientTest(primary);
+            PerformOneClientTest(node);
 
-            LogNodeStatus(primary);
+            LogNodeStatus(node);
         }
 
         private void PerformOneClientTest(ICodexNode primary)
@@ -23,6 +29,8 @@ namespace CodexTests.BasicTests
 
             var contentId = primary.UploadFile(testFile);
 
+            AssertNodesContainFile(contentId, primary);
+
             var downloadedFile = primary.DownloadContent(contentId);
 
             testFile.AssertIsEqual(downloadedFile);

Tests/CodexReleaseTests/DataTests/StreamlessDownloadTest.cs

@@ -0,0 +1,47 @@
+using CodexTests;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class StreamlessDownloadTest : CodexDistTest
+    {
+        [Test]
+        public void StreamlessTest()
+        {
+            var uploader = StartCodex();
+            var downloader = StartCodex(s => s.WithBootstrapNode(uploader));
+
+            var file = GenerateTestFile(10.MB());
+            var size = file.GetFilesize().SizeInBytes;
+            var cid = uploader.UploadFile(file);
+
+            var startSpace = downloader.Space();
+            var start = DateTime.UtcNow;
+            var localDataset = downloader.DownloadStreamless(cid);
+
+            Assert.That(localDataset.Cid, Is.EqualTo(cid));
+            Assert.That(localDataset.Manifest.OriginalBytes.SizeInBytes, Is.EqualTo(file.GetFilesize().SizeInBytes));
+
+            // TODO: We have no way to inspect the status or progress of the download.
+            // We use local space information to estimate.
+            var retry = new Retry("Checking local space",
+                maxTimeout: TimeSpan.FromMinutes(2),
+                sleepAfterFail: TimeSpan.FromSeconds(3),
+                onFail: f => { });
+
+            retry.Run(() =>
+            {
+                var space = downloader.Space();
+                var expected = startSpace.FreeBytes - size;
+                if (space.FreeBytes > expected) throw new Exception("Expected free space not reached.");
+            });
+
+            // Stop the uploader node and verify that the downloader has the data.
+            uploader.Stop(waitTillStopped: true);
+            var downloaded = downloader.DownloadContent(cid);
+            file.AssertIsEqual(downloaded);
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/SwarmTest.cs

@@ -0,0 +1,198 @@
+using CodexPlugin;
+using CodexTests;
+using FileUtils;
+using Logging;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class SwarmTests : CodexDistTest
+    {
+        [Test]
+        [Ignore("a")]
+        [Combinatorial]
+        public void SmallSwarm(
+            [Values(true, false)] bool peerImage,
+            [Values(2, 5, 10)] int NumberOfNodes,
+            [Values(2, 5, 10)] int FileSizeMb
+        )
+        {
+            // "thatbenbierens/nim-codex:peerselect2"
+            // "codexstorage/nim-codex:latest-dist-tests"
+
+            if (peerImage)
+            {
+                CodexContainerRecipe.DockerImageOverride = "thatbenbierens/nim-codex:peerselect2";
+            }
+            else
+            {
+                CodexContainerRecipe.DockerImageOverride = "codexstorage/nim-codex:0.1.9-dist-tests";
+            }
+
+            var boot = StartCodex(s => s.WithName("Bootstrap"));
+
+            var nodes = StartCodex(NumberOfNodes, s => s.WithBootstrapNode(boot));
+            var files = nodes.Select(n => UploadUniqueFilePerNode(n, FileSizeMb)).ToArray();
+
+            var tasks = ParallelDownloadEachFile(nodes, files);
+            Task.WaitAll(tasks);
+
+            AssertAllFilesDownloadedCorrectly(files);
+        }
+
+        [Test]
+        [Combinatorial]
+        public void Spppeeeed(
+            [Values(
+            23
+            )] int idx
+        )
+        {
+            string[] imgs = [
+                            //"thatbenbierens/nim-codex:peerselect2", // S review comments
+            "thatbenbierens/nim-codex:peerselect3", // X send wrong wantblock
+            "thatbenbierens/nim-codex:peerselect4", // S send wrong wantblock
+            "thatbenbierens/nim-codex:peerselect5", // X old task handler
+            "thatbenbierens/nim-codex:peerselect6", // S track only wantblocks
+            "thatbenbierens/nim-codex:peerselect7", // S old request proc
+            "thatbenbierens/nim-codex:peerselect8", // F! 3607b88 - sends wantBlock to peers with block. wantHave to everyone else
+            
+            "thatbenbierens/nim-codex:peerselect9", // [6] F + Slowing! 64e691b - Fixes issue where peerWants are only stored for type wantBlock.
+            //[2024-12-05T15:38:32.3332175Z] DL(12 secs)
+            //[2024-12-05T15:38:58.6083189Z] DL(20 secs)
+            //[2024-12-05T15:39:36.2547136Z] DL(29 secs)
+            //[2024-12-05T15:40:27.0061893Z] DL(38 secs)
+            //[2024-12-05T15:41:27.8224210Z] DL(47 secs)
+
+            "thatbenbierens/nim-codex:peerselect9log2", // S [7] same, var schedule + checking peerwants list length
+            "thatbenbierens/nim-codex:peerselect9log7", // S but stable [8] same, var schedule + checking peerwants list length
+            "thatbenbierens/nim-codex:peerselect9log8", // S [9] patch for not storing cancels + ed3e91c - Review comments by Dmitriy
+            "thatbenbierens/nim-codex:peerselect9log9", // S [10] ref object
+            "thatbenbierens/nim-codex:peerselect9log10", // S [11] presencecheck only new wants
+            "thatbenbierens/nim-codex:peerselect9log11", // S [12] same but no metrics + trace "wantList.entries.len" == always 1
+            "thatbenbierens/nim-codex:peerselect9log12", // S [13] always schedule peer
+            "thatbenbierens/nim-codex:peerselect9log13", // F! [14] 64e691b + new entry add if not e.cancel
+            "thatbenbierens/nim-codex:peerselect9log14", // F [15] ed3e91c + proc wantListHandler from "64e691b + new entry add if not e.cancel"
+            "thatbenbierens/nim-codex:peerselect9log15", // F [16] same, move metrics up
+            "thatbenbierens/nim-codex:peerselect9log16", // F [17] 1f063fe (branchlatest) proc wantlisthandler from previous
+            "thatbenbierens/nim-codex:peerselect9log17", // F [18] prev + restore schedulePeer bool
+            "thatbenbierens/nim-codex:peerselect9log18", // S [19] newcommit + moves presence check behind !cancel + type = wantHave
+            "thatbenbierens/nim-codex:peerselect9log19", // S [20] newcommit + moves presence check behind !cancel
+
+            "thatbenbierens/nim-codex:peerselect9log20", // F! [21] newcommit + moves presence check behind type == wanthave
+            "thatbenbierens/nim-codex:peerselect9log22", // ? [22] same + logging + logging
+            "thatbenbierens/nim-codex:peerselect9log23", // ? [23] same + logging + logging intentionally broken to compare!
+
+
+
+            "codexstorage/nim-codex:0.1.9-dist-tests", // F
+            "codexstorage/nim-codex:sha-8e29939-dist-tests", // F 8e29939 - Send pluralized wantBlock messages (#1016)
+            "codexstorage/nim-codex:sha-2124996-dist-tests"  // F 2124996 - Requesting the same CID sometimes causes a worker to discard the request if it's already inflight by another worker. (#1002)
+                ];
+
+            var img = imgs[idx];
+
+            CodexContainerRecipe.DockerImageOverride = img;
+
+            var boot = StartCodex(s => s.WithName("Bootstrap"));
+            var uploader = StartCodex(s => s.WithName("Uploader").WithBootstrapNode(boot));
+            var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(boot));
+
+            var total = TimeSpan.Zero;
+            var number = 1;
+
+            for (var i = 0; i < number; i++)
+            {
+                var file = GenerateTestFile(100.MB());
+                var cid = uploader.UploadFile(file);
+
+                var duration = Stopwatch.Measure(GetTestLog(), "DL", () =>
+                {
+                    downloader.DownloadContent(cid);
+                }) ;
+                
+                total += duration;
+                if (duration.TotalMinutes > 1.0) Assert.Fail("too slow");
+            }
+
+            var avg = total / number;
+            Log($"{img} 100MB download average duration: {avg}");
+        }
+
+        private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node, int fileSizeMb)
+        {
+            var file = GenerateTestFile(fileSizeMb.MB());
+            var cid = node.UploadFile(file);
+            return new SwarmTestNetworkFile(file, cid);
+        }
+
+        private Task[] ParallelDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files)
+        {
+            var tasks = new List<Task>();
+
+            foreach (var node in nodes)
+            {
+                tasks.Add(StartDownload(node, files));
+            }
+
+            return tasks.ToArray();
+        }
+
+        private Task StartDownload(ICodexNode node, SwarmTestNetworkFile[] files)
+        {
+            return Task.Run(() =>
+            {
+                var remaining = files.ToList();
+
+                while (remaining.Count > 0)
+                {
+                    var file = remaining.PickOneRandom();
+                    try
+                    {
+                        var dl = node.DownloadContent(file.Cid);
+                        lock (file.Lock)
+                        {
+                            file.Downloaded.Add(dl);
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        file.Error = ex;
+                    }
+                }
+            });
+        }
+
+        private void AssertAllFilesDownloadedCorrectly(SwarmTestNetworkFile[] files)
+        {
+            foreach (var file in files)
+            {
+                if (file.Error != null) throw file.Error;
+                lock (file.Lock)
+                {
+                    foreach (var dl in file.Downloaded)
+                    {
+                        file.Original.AssertIsEqual(dl);
+                    }
+                }
+            }
+        }
+
+        private class SwarmTestNetworkFile
+        {
+            public SwarmTestNetworkFile(TrackedFile original, ContentId cid)
+            {
+                Original = original;
+                Cid = cid;
+            }
+
+            public TrackedFile Original { get; }
+            public ContentId Cid { get; }
+            public object Lock { get; } = new object();
+            public List<TrackedFile?> Downloaded { get; } = new List<TrackedFile?>();
+            public Exception? Error { get; set; } = null;
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/ThreeClientTest.cs

@@ -0,0 +1,32 @@
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    public class ThreeClientTest : AutoBootstrapDistTest
+    {
+        [Test]
+        public void ThreeClient()
+        {
+            var primary = StartCodex();
+            var secondary = StartCodex();
+
+            var testFile = GenerateTestFile(10.MB());
+
+            var contentId = primary.UploadFile(testFile);
+            AssertNodesContainFile(contentId, primary);
+
+            var downloadedFile = secondary.DownloadContent(contentId);
+            AssertNodesContainFile(contentId, primary, secondary);
+
+            testFile.AssertIsEqual(downloadedFile);
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/TwoClientTest.cs

@@ -1,8 +1,14 @@
 using CodexPlugin;
+using CodexTests;
 using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
 using Utils;
 
-namespace CodexTests.BasicTests
+namespace CodexReleaseTests.DataTests
 {
     [TestFixture]
     public class TwoClientTests : CodexDistTest
@@ -17,6 +23,7 @@ namespace CodexTests.BasicTests
         }
 
         [Test]
+        [Ignore("Location selection is currently unavailable.")]
         public void TwoClientsTwoLocationsTest()
         {
             var locations = Ci.GetKnownLocations();
@@ -42,8 +49,10 @@ namespace CodexTests.BasicTests
             var testFile = GenerateTestFile(size);
 
             var contentId = uploader.UploadFile(testFile);
+            AssertNodesContainFile(contentId, uploader);
 
             var downloadedFile = downloader.DownloadContent(contentId);
+            AssertNodesContainFile(contentId, uploader, downloader);
 
             testFile.AssertIsEqual(downloadedFile);
             CheckLogForErrors(uploader, downloader);

Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs

@@ -0,0 +1,38 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class UnknownCidTest : CodexDistTest
+    {
+        [Test]
+        public void DownloadingUnknownCidDoesNotCauseCrash()
+        {
+            var node = StartCodex();
+
+            var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ");
+
+            var localFiles = node.LocalFiles().Content;
+            CollectionAssert.DoesNotContain(localFiles.Select(f => f.Cid), unknownCid);
+
+            try
+            {
+                node.DownloadContent(unknownCid);
+            }
+            catch (Exception ex)
+            {
+                var expectedMessage = $"Download of '{unknownCid.Id}' timed out";
+                if (!ex.Message.StartsWith(expectedMessage)) throw;
+            }
+
+            WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), node);
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs

@@ -0,0 +1,20 @@
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodexReleaseTests.MarketTests
+{
+    public class ContractFailedTest : CodexDistTest
+    {
+        [Test]
+        [Ignore("TODO - Test in which hosts are punished for failing a contract")]
+        public void ContractFailed()
+        {
+
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/ContractRepairedTest.cs

@@ -0,0 +1,18 @@
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodexReleaseTests.MarketTests
+{
+    public class ContractRepairedTest
+    {
+        [Test]
+        [Ignore("TODO - Test in which a host fails, but the slot is repaired")]
+        public void ContractRepaired()
+        {
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs

@@ -0,0 +1,73 @@
+using CodexContractsPlugin;
+using CodexPlugin;
+using GethPlugin;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    [TestFixture]
+    public class ContractSuccessfulTest : MarketplaceAutoBootstrapDistTest
+    {
+        private const int FilesizeMb = 10;
+
+        protected override int NumberOfHosts => 4;
+        protected override int NumberOfClients => 1;
+        protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB();
+        protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration();
+        private readonly TestToken pricePerSlotPerSecond = 10.TstWei();
+
+        [Test]
+        public void ContractSuccessful()
+        {
+            var hosts = StartHosts();
+            var client = StartClients().Single();
+
+            var request = CreateStorageRequest(client);
+
+            request.WaitForStorageContractSubmitted();
+            AssertContractIsOnChain(request);
+
+            request.WaitForStorageContractStarted();
+            AssertContractSlotsAreFilledByHosts(request, hosts);
+
+            request.WaitForStorageContractFinished(GetContracts());
+
+            AssertClientHasPaidForContract(pricePerSlotPerSecond, client, request, hosts);
+            AssertHostsWerePaidForContract(pricePerSlotPerSecond, request, hosts);
+            AssertHostsCollateralsAreUnchanged(hosts);
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
+        {
+            var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
+            var config = GetContracts().Deployment.Config;
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = GetContractDuration(),
+                Expiry = GetContractExpiry(),
+                MinRequiredNumberOfNodes = (uint)NumberOfHosts,
+                NodeFailureTolerance = (uint)(NumberOfHosts / 2),
+                PricePerSlotPerSecond = pricePerSlotPerSecond,
+                ProofProbability = 20,
+                RequiredCollateral = 1.Tst()
+            });
+        }
+
+        private TimeSpan GetContractExpiry()
+        {
+            return GetContractDuration() / 2;
+        }
+
+        private TimeSpan GetContractDuration()
+        {
+            return Get8TimesConfiguredPeriodDuration() / 2;
+        }
+
+        private TimeSpan Get8TimesConfiguredPeriodDuration()
+        {
+            var config = GetContracts().Deployment.Config;
+            return TimeSpan.FromSeconds(((double)config.Proofs.Period) * 8.0);
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs

@@ -0,0 +1,254 @@
+using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
+using CodexPlugin;
+using CodexTests;
+using DistTestCore;
+using GethPlugin;
+using Nethereum.Hex.HexConvertors.Extensions;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest
+    {
+        private readonly Dictionary<TestLifecycle, MarketplaceHandle> handles = new Dictionary<TestLifecycle, MarketplaceHandle>();
+        protected const int StartingBalanceTST = 1000;
+        protected const int StartingBalanceEth = 10;
+
+        protected override void LifecycleStart(TestLifecycle lifecycle)
+        {
+            base.LifecycleStart(lifecycle);
+            var geth = Ci.StartGethNode(s => s.IsMiner());
+            var contracts = Ci.StartCodexContracts(geth);
+            handles.Add(lifecycle, new MarketplaceHandle(geth, contracts));
+        }
+
+        protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result)
+        {
+            base.LifecycleStop(lifecycle, result);
+            handles.Remove(lifecycle);
+        }
+
+        protected IGethNode GetGeth()
+        {
+            return handles[Get()].Geth;
+        }
+
+        protected ICodexContracts GetContracts()
+        {
+            return handles[Get()].Contracts;
+        }
+
+        protected abstract int NumberOfHosts { get; }
+        protected abstract int NumberOfClients { get; }
+        protected abstract ByteSize HostAvailabilitySize { get; }
+        protected abstract TimeSpan HostAvailabilityMaxDuration { get; }
+
+        public ICodexNodeGroup StartHosts()
+        {
+            var hosts = StartCodex(NumberOfHosts, s => s
+                .WithName("host")
+                .EnableMarketplace(GetGeth(), GetContracts(), m => m
+                    .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
+                    .AsStorageNode()
+                )
+            );
+
+            var config = GetContracts().Deployment.Config;
+            foreach (var host in hosts)
+            {
+                Assert.That(GetTstBalance(host).TstWei, Is.EqualTo(StartingBalanceTST.Tst().TstWei));
+                Assert.That(GetEthBalance(host).Wei, Is.EqualTo(StartingBalanceEth.Eth().Wei));
+
+                host.Marketplace.MakeStorageAvailable(new CodexPlugin.StorageAvailability(
+                    totalSpace: HostAvailabilitySize,
+                    maxDuration: HostAvailabilityMaxDuration,
+                    minPriceForTotalSpace: 1.TstWei(),
+                    maxCollateral: 999999.Tst())
+                );
+            }
+            return hosts;
+        }
+
+        public TestToken GetTstBalance(ICodexNode node)
+        {
+            return GetContracts().GetTestTokenBalance(node);
+        }
+
+        public TestToken GetTstBalance(EthAddress address)
+        {
+            return GetContracts().GetTestTokenBalance(address);
+        }
+
+        public Ether GetEthBalance(ICodexNode node)
+        {
+            return GetGeth().GetEthBalance(node);
+        }
+
+        public Ether GetEthBalance(EthAddress address)
+        {
+            return GetGeth().GetEthBalance(address);
+        }
+
+        public ICodexNodeGroup StartClients()
+        {
+            return StartCodex(NumberOfClients, s => s
+                .WithName("client")
+                .EnableMarketplace(GetGeth(), GetContracts(), m => m
+                    .WithInitial(StartingBalanceEth.Eth(), StartingBalanceTST.Tst())
+                )
+            );
+        }
+
+        public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts, string purchaseId)
+        {
+            var fills = GetOnChainSlotFills(possibleHosts);
+            return fills.Where(f => f
+                .SlotFilledEvent.RequestId.ToHex(false).ToLowerInvariant() == purchaseId.ToLowerInvariant())
+                .ToArray();
+        }
+
+        public SlotFill[] GetOnChainSlotFills(ICodexNodeGroup possibleHosts)
+        {
+            var events = GetContracts().GetEvents(GetTestRunTimeRange());
+            var fills = events.GetSlotFilledEvents();
+            return fills.Select(f =>
+            {
+                var host = possibleHosts.Single(h => h.EthAddress.Address == f.Host.Address);
+                return new SlotFill(f, host);
+
+            }).ToArray();
+        }
+
+        protected void AssertClientHasPaidForContract(TestToken pricePerSlotPerSecond, ICodexNode client, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
+        {
+            var balance = GetTstBalance(client);
+            var expectedBalance = StartingBalanceTST.Tst() - GetContractFinalCost(pricePerSlotPerSecond, contract, hosts);
+
+            Assert.That(balance, Is.EqualTo(expectedBalance), "Client balance incorrect.");
+        }
+
+        protected void AssertHostsWerePaidForContract(TestToken pricePerSlotPerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
+        {
+            var fills = GetOnChainSlotFills(hosts);
+            var submitUtc = GetContractOnChainSubmittedUtc(contract);
+            var finishUtc = submitUtc + contract.Purchase.Duration;
+            var expectedBalances = new Dictionary<EthAddress, TestToken>();
+            foreach (var host in hosts) expectedBalances.Add(host.EthAddress, StartingBalanceTST.Tst());
+            foreach (var fill in fills)
+            {
+                var slotDuration = finishUtc - fill.SlotFilledEvent.Block.Utc;
+                expectedBalances[fill.Host.EthAddress] += GetContractCostPerSlot(pricePerSlotPerSecond, slotDuration);
+            }
+
+            foreach (var pair in expectedBalances)
+            {
+                var balance = GetTstBalance(pair.Key);
+                Assert.That(balance, Is.EqualTo(pair.Value), "Host was not paid for storage.");
+            }
+        }
+
+        protected void AssertHostsCollateralsAreUnchanged(ICodexNodeGroup hosts)
+        {
+            // There is no separate collateral location yet.
+            // All host balances should be equal to or greater than the starting balance.
+            foreach (var host in hosts)
+            {
+                Assert.That(GetTstBalance(host), Is.GreaterThanOrEqualTo(StartingBalanceTST.Tst()));
+            }
+        }
+
+        private TestToken GetContractFinalCost(TestToken pricePerSlotPerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts)
+        {
+            var fills = GetOnChainSlotFills(hosts);
+            var result = 0.Tst();
+            var submitUtc = GetContractOnChainSubmittedUtc(contract);
+            var finishUtc = submitUtc + contract.Purchase.Duration;
+
+            foreach (var fill in fills)
+            {
+                var slotDuration = finishUtc - fill.SlotFilledEvent.Block.Utc;
+                result += GetContractCostPerSlot(pricePerSlotPerSecond, slotDuration);
+            }
+
+            return result;
+        }
+
+        private DateTime GetContractOnChainSubmittedUtc(IStoragePurchaseContract contract)
+        {
+            var events = GetContracts().GetEvents(GetTestRunTimeRange());
+            var submitEvent = events.GetStorageRequests().Single(e => e.RequestId.ToHex(false) == contract.PurchaseId);
+            return submitEvent.Block.Utc;
+        }
+
+        private TestToken GetContractCostPerSlot(TestToken pricePerSlotPerSecond, TimeSpan slotDuration)
+        {
+            return pricePerSlotPerSecond * (int)slotDuration.TotalSeconds;
+        }
+
+        protected void AssertContractSlotsAreFilledByHosts(IStoragePurchaseContract contract, ICodexNodeGroup hosts)
+        {
+            var activeHosts = new Dictionary<int, SlotFill>();
+
+            Time.Retry(() =>
+            {
+                var fills = GetOnChainSlotFills(hosts, contract.PurchaseId);
+                foreach (var fill in fills)
+                {
+                    var index = (int)fill.SlotFilledEvent.SlotIndex;
+                    if (!activeHosts.ContainsKey(index))
+                    {
+                        activeHosts.Add(index, fill);
+                    }
+                }
+
+                if (activeHosts.Count != contract.Purchase.MinRequiredNumberOfNodes) throw new Exception("Not all slots were filled...");
+
+            }, nameof(AssertContractSlotsAreFilledByHosts));
+        }
+
+        protected void AssertContractIsOnChain(IStoragePurchaseContract contract)
+        {
+            AssertOnChainEvents(events =>
+            {
+                var onChainRequests = events.GetStorageRequests();
+                if (onChainRequests.Any(r => r.Id == contract.PurchaseId)) return;
+                throw new Exception($"OnChain request {contract.PurchaseId} not found...");
+            }, nameof(AssertContractIsOnChain));
+        }
+
+        protected void AssertOnChainEvents(Action<ICodexContractsEvents> onEvents, string description)
+        {
+            Time.Retry(() =>
+            {
+                var events = GetContracts().GetEvents(GetTestRunTimeRange());
+                onEvents(events);
+            }, description);
+        }
+
+        public class SlotFill
+        {
+            public SlotFill(SlotFilledEventDTO slotFilledEvent, ICodexNode host)
+            {
+                SlotFilledEvent = slotFilledEvent;
+                Host = host;
+            }
+
+            public SlotFilledEventDTO SlotFilledEvent { get; }
+            public ICodexNode Host { get; }
+        }
+
+        private class MarketplaceHandle
+        {
+            public MarketplaceHandle(IGethNode geth, ICodexContracts contracts)
+            {
+                Geth = geth;
+                Contracts = contracts;
+            }
+
+            public IGethNode Geth { get; }
+            public ICodexContracts Contracts { get; }
+        }
+    }
+}

Tests/CodexReleaseTests/MarketTests/MultipleContractsTest.cs

@@ -0,0 +1,83 @@
+using CodexContractsPlugin;
+using CodexPlugin;
+using GethPlugin;
+using NUnit.Framework;
+using Utils;
+
+namespace CodexReleaseTests.MarketTests
+{
+    [TestFixture]
+    public class MultipleContractsTest : MarketplaceAutoBootstrapDistTest
+    {
+        private const int FilesizeMb = 10;
+
+        protected override int NumberOfHosts => 8;
+        protected override int NumberOfClients => 3;
+        protected override ByteSize HostAvailabilitySize => (5 * FilesizeMb).MB();
+        protected override TimeSpan HostAvailabilityMaxDuration => Get8TimesConfiguredPeriodDuration();
+        private readonly TestToken pricePerSlotPerSecond = 10.TstWei();
+
+        [Test]
+        [Ignore("TODO - Test where multiple successful contracts are run simultaenously")]
+        public void MultipleSuccessfulContracts()
+        {
+            var hosts = StartHosts();
+            var clients = StartClients();
+
+            var requests = clients.Select(c => CreateStorageRequest(c)).ToArray();
+
+            All(requests, r =>
+            {
+                r.WaitForStorageContractSubmitted();
+                AssertContractIsOnChain(r);
+            });
+
+            All(requests, r => r.WaitForStorageContractStarted());
+            All(requests, r => AssertContractSlotsAreFilledByHosts(r, hosts));
+
+            All(requests, r => r.WaitForStorageContractFinished(GetContracts()));
+
+            // todo:
+            //AssertClientHasPaidForContract(pricePerSlotPerSecond, client, request, hosts);
+            //AssertHostsWerePaidForContract(pricePerSlotPerSecond, request, hosts);
+            //AssertHostsCollateralsAreUnchanged(hosts);
+        }
+
+        private void All(IStoragePurchaseContract[] requests, Action<IStoragePurchaseContract> action)
+        {
+            foreach (var r in requests) action(r);
+        }
+
+        private IStoragePurchaseContract CreateStorageRequest(ICodexNode client)
+        {
+            var cid = client.UploadFile(GenerateTestFile(FilesizeMb.MB()));
+            var config = GetContracts().Deployment.Config;
+            return client.Marketplace.RequestStorage(new StoragePurchaseRequest(cid)
+            {
+                Duration = GetContractDuration(),
+                Expiry = GetContractExpiry(),
+                MinRequiredNumberOfNodes = (uint)NumberOfHosts,
+                NodeFailureTolerance = (uint)(NumberOfHosts / 2),
+                PricePerSlotPerSecond = pricePerSlotPerSecond,
+                ProofProbability = 20,
+                RequiredCollateral = 1.Tst()
+            });
+        }
+
+        private TimeSpan GetContractExpiry()
+        {
+            return GetContractDuration() / 2;
+        }
+
+        private TimeSpan GetContractDuration()
+        {
+            return Get8TimesConfiguredPeriodDuration() / 2;
+        }
+
+        private TimeSpan Get8TimesConfiguredPeriodDuration()
+        {
+            var config = GetContracts().Deployment.Config;
+            return TimeSpan.FromSeconds(((double)config.Proofs.Period) * 8.0);
+        }
+    }
+}

Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs

@@ -0,0 +1,62 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexReleaseTests.NodeTests
+{
+    [TestFixture]
+    public class BasicInfoTests : CodexDistTest
+    {
+        [Test]
+        public void QuotaTest()
+        {
+            var size = 3.GB();
+            var node = StartCodex(s => s.WithStorageQuota(size));
+            var space = node.Space();
+
+            Assert.That(space.QuotaMaxBytes, Is.EqualTo(size.SizeInBytes));
+        }
+
+        [Test]
+        public void Spr()
+        {
+            var node = StartCodex();
+            
+            var info = node.GetDebugInfo();
+            Assert.That(!string.IsNullOrEmpty(info.Spr));
+
+            var spr = node.GetSpr();
+            Assert.That(!string.IsNullOrEmpty(spr));
+
+            Assert.That(info.Spr, Is.EqualTo(spr));
+        }
+
+        [Test]
+        public void VersionInfo()
+        {
+            var node = StartCodex();
+
+            var info = node.GetDebugInfo();
+            Assert.That(!string.IsNullOrEmpty(info.Version.Version));
+            Assert.That(!string.IsNullOrEmpty(info.Version.Revision));
+        }
+
+        [Test]
+        public void AnnounceAddress()
+        {
+            var node = StartCodex();
+            var addr = node.Container.GetInternalAddress(CodexContainerRecipe.ListenPortTag);
+
+            var info = node.GetDebugInfo();
+
+            Assert.That(info.AnnounceAddresses.Count, Is.GreaterThan(0));
+            // Ideally we'd assert the pod IP is in the announce address, but we can't access it from here.
+        }
+    }
+}

Tests/CodexReleaseTests/NodeTests/PeerTableTests.cs

@@ -0,0 +1,31 @@
+using CodexPlugin;
+using CodexTests;
+using CodexTests.Helpers;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexReleaseTests.NodeTests
+{
+    [TestFixture]
+    public class PeerTableTests : AutoBootstrapDistTest
+    {
+        [Test]
+        public void PeerTableCompleteness()
+        {
+            var nodes = StartCodex(10);
+
+            AssertAllNodesSeeEachOther(nodes.Concat([BootstrapNode!]));
+        }
+
+        private void AssertAllNodesSeeEachOther(IEnumerable<ICodexNode> nodes)
+        {
+            var helper = new PeerConnectionTestHelpers(GetTestLog());
+            helper.AssertFullyConnected(nodes);
+        }
+    }
+}

Tests/CodexReleaseTests/Parallelism.cs

@@ -0,0 +1,6 @@
+using NUnit.Framework;
+
+[assembly: LevelOfParallelism(1)]
+namespace CodexReleaseTests.DataTests
+{
+}

Tests/CodexTests/AutoBootstrapDistTest.cs

@@ -1,28 +0,0 @@
-using CodexPlugin;
-using NUnit.Framework;
-
-namespace CodexTests
-{
-    public class AutoBootstrapDistTest : CodexDistTest
-    {
-        [SetUp]
-        public void SetUpBootstrapNode()
-        {
-            BootstrapNode = StartCodex(s => s.WithName("BOOTSTRAP"));
-        }
-
-        [TearDown]
-        public void TearDownBootstrapNode()
-        {
-            BootstrapNode = null;
-        }
-
-        protected override void OnCodexSetup(ICodexSetup setup)
-        {
-            if (BootstrapNode != null) setup.WithBootstrapNode(BootstrapNode);
-        }
-
-        protected ICodexNode? BootstrapNode { get; private set; }
-        
-    }
-}

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -1,72 +0,0 @@
-using CodexPlugin;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.BasicTests
-{
-    [TestFixture]
-    public class ThreeClientTest : AutoBootstrapDistTest
-    {
-        [Test]
-        public void ThreeClient()
-        {
-            var primary = StartCodex();
-            var secondary = StartCodex();
-
-            var testFile = GenerateTestFile(10.MB());
-
-            var contentId = primary.UploadFile(testFile);
-
-            var downloadedFile = secondary.DownloadContent(contentId);
-
-            testFile.AssertIsEqual(downloadedFile);
-        }
-
-        [Test]
-        [CreateTranscript(nameof(SwarmTest))]
-        public void SwarmTest()
-        {
-            var uploader = StartCodex(s => s.WithName("uploader"));
-            var downloaders = StartCodex(5, s => s.WithName("downloader"));
-
-            var file = GenerateTestFile(100.MB());
-            var cid = uploader.UploadFile(file);
-
-            var result = Parallel.ForEach(downloaders, d =>
-            {
-                d.DownloadContent(cid);
-            });
-
-            Assert.That(result.IsCompleted);
-        }
-
-        [Test]
-        public void DownloadingUnknownCidDoesNotCauseCrash()
-        {
-            var node = StartCodex(2).First();
-
-            var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ");
-
-            try
-            {
-                node.DownloadContent(unknownCid);
-            }
-            catch (Exception ex)
-            {
-                if (!ex.Message.StartsWith("Retry 'DownloadFile' timed out"))
-                {
-                    throw;
-                }
-            }
-
-            // Check that the node stays alive for at least another 5 minutes.
-            var start = DateTime.UtcNow;
-            while ((DateTime.UtcNow - start) < TimeSpan.FromMinutes(5))
-            {
-                Thread.Sleep(5000);
-                var info = node.GetDebugInfo();
-                Assert.That(!string.IsNullOrEmpty(info.Id));
-            }
-        }
-    }
-}

Tests/DistTestCore/DistTestCore.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>DistTestCore</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>

Tests/DistTestCore/DownloadedLogExtensions.cs

@@ -23,5 +23,21 @@ namespace DistTestCore
             }
             CollectionAssert.IsEmpty(errors);
         }
+
+        public static void AssertLogDoesNotContainLinesStartingWith(this IDownloadedLog log, params string[] unexpectedStrings)
+        {
+            var errors = new List<string>();
+            log.IterateLines(line =>
+            {
+                foreach (var str in unexpectedStrings)
+                {
+                    if (line.StartsWith(str))
+                    {
+                        errors.Add($"Found '{str}' at start of line '{line}'.");
+                    }
+                }
+            });
+            CollectionAssert.IsEmpty(errors);
+        }
     }
 }

Tests/DistTestCore/TestLifecycle.cs

@@ -22,6 +22,7 @@ namespace DistTestCore
             Log = log;
             Configuration = configuration;
             TimeSet = timeSet;
+            TestNamespace = testNamespace;
             TestStart = DateTime.UtcNow;
 
             entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet);
@@ -36,6 +37,7 @@ namespace DistTestCore
         public TestLog Log { get; }
         public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
+        public string TestNamespace { get; }
         public bool WaitForCleanup { get; }
         public CoreInterface CoreInterface { get; }
 

Tests/ExperimentalTests/AutoBootstrapDistTest.cs

@@ -0,0 +1,46 @@
+using CodexPlugin;
+using DistTestCore;
+using NUnit.Framework;
+
+namespace CodexTests
+{
+    public class AutoBootstrapDistTest : CodexDistTest
+    {
+        private readonly Dictionary<TestLifecycle, ICodexNode> bootstrapNodes = new Dictionary<TestLifecycle, ICodexNode>();
+
+        [SetUp]
+        public void SetUpBootstrapNode()
+        {
+            var tl = Get();
+            if (!bootstrapNodes.ContainsKey(tl))
+            {
+                bootstrapNodes.Add(tl, StartCodex(s => s.WithName("BOOTSTRAP_" + tl.TestNamespace)));
+            }
+        }
+
+        [TearDown]
+        public void TearDownBootstrapNode()
+        {
+            bootstrapNodes.Remove(Get());
+        }
+
+        protected override void OnCodexSetup(ICodexSetup setup)
+        {
+            var node = BootstrapNode;
+            if (node != null) setup.WithBootstrapNode(node);
+        }
+
+        protected ICodexNode? BootstrapNode
+        {
+            get
+            {
+                var tl = Get();
+                if (bootstrapNodes.TryGetValue(tl, out var node))
+                {
+                    return node;
+                }
+                return null;
+            }
+        }
+    }
+}

Tests/ExperimentalTests/BasicTests/AsyncProfiling.cs

@@ -0,0 +1,85 @@
+using NUnit.Framework;
+using MetricsPlugin;
+using Utils;
+
+namespace CodexTests.BasicTests
+{
+    [TestFixture]
+    public class AsyncProfiling : CodexDistTest
+    {
+        [Test]
+        public void AsyncProfileMetricsPlz()
+        {
+            var node = StartCodex(s => s.EnableMetrics());
+            var metrics = Ci.GetMetricsFor(scrapeInterval: TimeSpan.FromSeconds(3.0), node).Single();
+
+            var file = GenerateTestFile(100.MB());
+            node.UploadFile(file);
+
+            Thread.Sleep(10000);
+
+            var profilerMetrics = new AsyncProfileMetrics(metrics.GetAllMetrics());
+
+            var log = GetTestLog();
+            log.Log($"{nameof(profilerMetrics.CallCount)} = {profilerMetrics.CallCount.Highest()}");
+            log.Log($"{nameof(profilerMetrics.ExecTime)} = {profilerMetrics.ExecTime.Highest()}");
+            log.Log($"{nameof(profilerMetrics.ExecTimeWithChildren)} = {profilerMetrics.ExecTimeWithChildren.Highest()}");
+            log.Log($"{nameof(profilerMetrics.SingleExecTimeMax)} = {profilerMetrics.SingleExecTimeMax.Highest()}");
+            log.Log($"{nameof(profilerMetrics.WallTime)} = {profilerMetrics.WallTime.Highest()}");
+        }
+    }
+
+    public class AsyncProfileMetrics
+    {
+        public AsyncProfileMetrics(Metrics metrics)
+        {
+            CallCount = CreateMetric(metrics, "chronos_call_count_total");
+            ExecTime = CreateMetric(metrics, "chronos_exec_time_total");
+            ExecTimeWithChildren = CreateMetric(metrics, "chronos_exec_time_with_children_total");
+            SingleExecTimeMax = CreateMetric(metrics, "chronos_single_exec_time_max");
+            WallTime = CreateMetric(metrics, "chronos_wall_time_total");
+        }
+
+        public AsyncProfileMetric CallCount { get; }
+        public AsyncProfileMetric ExecTime { get; }
+        public AsyncProfileMetric ExecTimeWithChildren { get; }
+        public AsyncProfileMetric SingleExecTimeMax { get; }
+        public AsyncProfileMetric WallTime { get; }
+
+        private static AsyncProfileMetric CreateMetric(Metrics metrics, string name)
+        {
+            var sets = metrics.Sets.Where(s => s.Name == name).ToArray();
+            return new AsyncProfileMetric(sets);
+        }
+    }
+
+    public class AsyncProfileMetric
+    {
+        private readonly MetricsSet[] metricsSets;
+
+        public AsyncProfileMetric(MetricsSet[] metricsSets)
+        {
+            this.metricsSets = metricsSets;
+        }
+
+        public MetricsSet Highest()
+        {
+            MetricsSet? result = null;
+            var highest = double.MinValue;
+            foreach (var metric in metricsSets)
+            {
+                foreach (var value in metric.Values)
+                {
+                    if (value.Value > highest)
+                    {
+                        highest = value.Value;
+                        result = metric;
+                    }
+                }
+            }
+
+            if (result == null) throw new Exception("None were highest");
+            return result;
+        }
+    }
+}

Tests/ExperimentalTests/BasicTests/ExampleTests.cs

@@ -36,7 +36,7 @@ namespace CodexTests.BasicTests
             var primary2 = group2[0];
             var secondary2 = group2[1];
 
-            var metrics = Ci.GetMetricsFor(primary, primary2);
+            var metrics = Ci.GetMetricsFor(scrapeInterval: TimeSpan.FromSeconds(10), primary, primary2);
 
             primary.ConnectToPeer(secondary);
             primary2.ConnectToPeer(secondary2);

Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs

@@ -97,57 +97,22 @@ namespace CodexTests.BasicTests
 
             purchaseContract.WaitForStorageContractStarted();
 
+            var availabilities = hosts.Select(h => h.Marketplace.GetAvailabilities()).ToArray();
+            if (availabilities.All(h => h.All(a => a.FreeSpace.SizeInBytes == a.TotalSpace.SizeInBytes)))
+            {
+                Assert.Fail("Host availabilities were not used.");
+            }
+
             var request = GetOnChainStorageRequest(contracts, geth);
             AssertStorageRequest(request, purchase, contracts, client);
             AssertContractSlot(contracts, request, 0);
 
-            purchaseContract.WaitForStorageContractFinished();
+            purchaseContract.WaitForStorageContractFinished(contracts);
 
             AssertBalance(contracts, client, Is.LessThan(clientInitialBalance), "Buyer was not charged for storage.");
             Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished));
         }
 
-        [Test]
-        [Ignore("Integrated into MarketplaceExample to speed up testing.")]
-        public void CanDownloadContentFromContractCid()
-        {
-            var fileSize = 10.MB();
-            var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth"));
-            var contracts = Ci.StartCodexContracts(geth);
-            var testFile = CreateFile(fileSize);
-
-            var client = StartCodex(s => s
-                .WithName("Client")
-                .EnableMarketplace(geth, contracts, m => m
-                    .WithInitial(10.Eth(), 10.Tst())));
-
-            var uploadCid = client.UploadFile(testFile);
-
-            var purchase = new StoragePurchaseRequest(uploadCid)
-            {
-                PricePerSlotPerSecond = 2.TstWei(),
-                RequiredCollateral = 10.TstWei(),
-                MinRequiredNumberOfNodes = 5,
-                NodeFailureTolerance = 2,
-                ProofProbability = 5,
-                Duration = TimeSpan.FromMinutes(5),
-                Expiry = TimeSpan.FromMinutes(4)
-            };
-
-            var purchaseContract = client.Marketplace.RequestStorage(purchase);
-            var contractCid = purchaseContract.ContentId;
-            Assert.That(uploadCid.Id, Is.Not.EqualTo(contractCid.Id));
-
-            // Download both from client.
-            testFile.AssertIsEqual(client.DownloadContent(uploadCid));
-            testFile.AssertIsEqual(client.DownloadContent(contractCid));
-
-            // Download both from another node.
-            var downloader = StartCodex(s => s.WithName("Downloader"));
-            testFile.AssertIsEqual(downloader.DownloadContent(uploadCid));
-            testFile.AssertIsEqual(downloader.DownloadContent(contractCid));
-        }
-
         private TrackedFile CreateFile(ByteSize fileSize)
         {
             var segmentSize = new ByteSize(fileSize.SizeInBytes / 4);

Tests/ExperimentalTests/BasicTests/PyramidTests.cs

@@ -17,7 +17,7 @@ namespace CodexTests.BasicTests
         public void PyramidTest()
         {
             var size = 5.MB();
-            var numberOfLayers = 4;
+            var numberOfLayers = 3;
 
             var bottomLayer = StartLayers(numberOfLayers);
 

Tests/ExperimentalTests/CodexDistTest.cs

@@ -99,7 +99,7 @@ namespace CodexTests
             var log = Ci.DownloadLog(node);
 
             log.AssertLogDoesNotContain("Block validation failed");
-            log.AssertLogDoesNotContain("ERR ");
+            log.AssertLogDoesNotContainLinesStartingWith("ERR ");
         }
 
         public void LogNodeStatus(ICodexNode node, IMetricsAccess? metrics = null)
@@ -108,6 +108,39 @@ namespace CodexTests
                 GetBasicNodeStatus(node));
         }
 
+        public void WaitAndCheckNodesStaysAlive(TimeSpan duration, ICodexNodeGroup nodes)
+        {
+            WaitAndCheckNodesStaysAlive(duration, nodes.ToArray());
+        }
+
+        public void WaitAndCheckNodesStaysAlive(TimeSpan duration, params ICodexNode[] nodes)
+        {
+            var start = DateTime.UtcNow;
+            while ((DateTime.UtcNow - start) < duration)
+            {
+                Thread.Sleep(5000);
+                foreach (var node in nodes)
+                {
+                    var info = node.GetDebugInfo();
+                    Assert.That(!string.IsNullOrEmpty(info.Id));
+                }
+            }
+        }
+
+        public void AssertNodesContainFile(ContentId cid, ICodexNodeGroup nodes)
+        {
+            AssertNodesContainFile(cid, nodes.ToArray());
+        }
+
+        public void AssertNodesContainFile(ContentId cid, params ICodexNode[] nodes)
+        {
+            foreach (var node in nodes)
+            {
+                var localDatasets = node.LocalFiles();
+                CollectionAssert.Contains(localDatasets.Content.Select(c => c.Cid), cid);
+            }
+        }
+
         private string GetBasicNodeStatus(ICodexNode node)
         {
             return JsonConvert.SerializeObject(node.GetDebugInfo(), Formatting.Indented) + Environment.NewLine +

Tests/ExperimentalTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs

@@ -0,0 +1,25 @@
+using NUnit.Framework;
+using Utils;
+
+namespace CodexTests.DownloadConnectivityTests
+{
+    [TestFixture]
+    public class DetectBlockRetransmitTest : AutoBootstrapDistTest
+    {
+        [Test]
+        [Combinatorial]
+        [CreateTranscript("swarm_retransmit")]
+        public void DetectBlockRetransmits(
+            [Values(1, 5, 10, 20)] int fileSize,
+            [Values(3, 5, 10, 20)] int numNodes
+        )
+        {
+            var nodes = StartCodex(numNodes);
+            var file = GenerateTestFile(fileSize.MB());
+            var cid = nodes[0].UploadFile(file);
+
+            var tasks = nodes.Select(n => Task.Run(() => n.DownloadContent(cid))).ToArray();
+            Task.WaitAll(tasks);
+        }
+    }
+}