Sets up tests for symmetric, asymmetric and binary speed checks

* Fix job status check

* Add a delay before get Kubernetes job status

.dockerignore

@@ -0,0 +1,30 @@
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md
+!**/.gitignore
+!.git/HEAD
+!.git/config
+!.git/packed-refs
+!.git/refs/heads/**

.gitattributes

@@ -0,0 +1,10 @@
+# Set default behavior to automatically normalize line endings.
+* text=auto
+
+# Force bash scripts to always use lf line endings so that if a repo is accessed
+# in Unix via a file share from Windows, the scripts will work.
+*.sh text eol=lf
+
+# Likewise, force cmd and batch scripts to always use crlf
+*.cmd text eol=crlf
+*.bat text eol=crlf

.github/workflows/docker-autoclient.yml

@@ -0,0 +1,26 @@
+name: Docker - AutoClient
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/AutoClient/**'
+      - '!Tools/AutoClient/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-autoclient.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/AutoClient/docker/Dockerfile
+      docker_repo: codexstorage/codex-autoclient
+    secrets: inherit
+

.github/workflows/docker-marketinsights.yml

@@ -0,0 +1,27 @@
+name: Docker - MarketInsights API
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/MarketInsights/**'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-marketinsights.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/MarketInsights/Dockerfile
+      docker_repo: codexstorage/codex-marketinsights
+    secrets: inherit
+

.github/workflows/run-continuous-tests.yaml

@@ -80,7 +80,7 @@ env:
   TESTS_TARGET_DURATION: 2d
   TESTS_FILTER: ""
   TESTS_CLEANUP: true
-  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  JOB_MANIFEST: docker/job-continuous-tests.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
   KUBE_VERSION: v1.28.2
 
@@ -146,7 +146,7 @@ jobs:
         if: false
         run: |
           # Variables
-          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to run a node
           duration=600
           namespace="${{ env.NAMESPACE }}"
           pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")

.github/workflows/run-dist-tests.yaml

@@ -32,7 +32,7 @@ env:
   NAMEPREFIX: d-tests-runner
   NAMESPACE: default
   COMMAND: dotnet test Tests/CodexTests
-  JOB_MANIFEST: docker/dist-tests-job.yaml
+  JOB_MANIFEST: docker/job-dist-tests.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
   KUBE_VERSION: v1.28.2
 

.github/workflows/run-release-tests.yaml

@@ -0,0 +1,122 @@
+name: Run Release Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+      workflow_source:
+        description: Workflow source
+        required: false
+        type: string
+        default: ''
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
+  TEST_TYPE: release-tests
+  NAMEPREFIX: r-tests
+  NAMESPACE: default
+  JOB_MANIFEST: docker/job-release-tests.yaml
+  COMMAND: dotnet test Tests/CodexReleaseTests
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.30.5
+
+
+jobs:
+  run_tests:
+    name: Run Release Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
+          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'${COMMAND}'"')" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job to run tests
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}"
+          echo "Runner pod: $(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}')"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+
+      - name: Show Runner logs
+        run: |
+          # Variables
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          namespace="${{ env.NAMESPACE }}"
+          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath='{.items[0].metadata.name}')
+
+          # Check Pod status
+          WAIT=120
+          SECONDS=0
+          sleep=1
+          while (( SECONDS < WAIT )); do
+            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}')
+            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in the $phase state - Get the logs"; break; } || { echo "Pod $pod is in the $phase state - Retry in $sleep second(s) / $((WAIT - SECONDS))"; }
+            sleep $sleep
+          done
+
+          # Get logs
+          while [[ $(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}') == "Running" ]]; do
+            echo "Show ${pod} logs ..."
+            echo "----"
+            kubectl logs $pod -n $namespace -f || true
+            sleep 1
+          done
+
+      - name: Set workflow status from job status
+        run: |
+          sleep 5
+          job_status=$(kubectl get jobs ${{ env.NAMEPREFIX }} -n ${{ env.NAMESPACE }} -o jsonpath='{.status.conditions[0].type}')
+          echo "Job status: $job_status"
+          if [[ "${job_status}" != "Complete" ]]; then exit 1; fi

.gitignore

@@ -1,4 +1,5 @@
 .vs
 obj
 bin
-.vscode
+.vscode
+Tools/AutoClient/datapath

Framework/ArgsUniform/ArgsUniform.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/BlockchainUtils/BlockCache.cs

@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockCache
     {

Framework/BlockchainUtils/BlockTimeEntry.cs

@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockTimeEntry
     {

Framework/BlockchainUtils/BlockTimeFinder.cs

@@ -1,6 +1,6 @@
 using Logging;
 
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockTimeFinder
     {
@@ -29,7 +29,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetHighestBlockNumberBefore(DateTime moment)
         {
             bounds.Initialize();
-            if (moment <= bounds.Genesis.Utc) return null;
+            if (moment < bounds.Genesis.Utc) return null;
+            if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
             if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
 
             return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
@@ -38,7 +39,8 @@ namespace NethereumWorkflow.BlockUtils
         public ulong? GetLowestBlockNumberAfter(DateTime moment)
         {
             bounds.Initialize();
-            if (moment >= bounds.Current.Utc) return null;
+            if (moment > bounds.Current.Utc) return null;
+            if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber;
             if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
 
             return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;

Framework/BlockchainUtils/BlockchainBounds.cs

@@ -1,5 +1,11 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
+    public interface IWeb3Blocks
+    {
+        ulong GetCurrentBlockNumber();
+        DateTime? GetTimestampForBlock(ulong blockNumber);
+    }
+
     public class BlockchainBounds
     {
         private readonly BlockCache cache;

Framework/BlockchainUtils/BlockchainUtils.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+	<ItemGroup>
+		<PackageReference Include="Nethereum.Web3" Version="4.14.0" />
+	</ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/BlockchainUtils/ConversionExtensions.cs

@@ -1,7 +1,7 @@
 using Nethereum.Hex.HexTypes;
 using System.Numerics;
 
-namespace NethereumWorkflow
+namespace BlockchainUtils
 {
     public static class ConversionExtensions
     {

Framework/Core/Core.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/Core/CoreInterface.cs

@@ -30,11 +30,7 @@ namespace Core
         public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
         {
             var workflow = entryPoint.Tools.CreateWorkflow();
-            var msg = $"Downloading container log for '{container.Name}'";
-            entryPoint.Tools.GetLog().Log(msg);
-            var logHandler = new WriteToFileLogHandler(entryPoint.Tools.GetLog(), msg);
-            workflow.DownloadContainerLog(container, logHandler, tailLines);
-            return new DownloadedLog(logHandler);
+            return workflow.DownloadContainerLog(container, tailLines);
         }
 
         public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)

Framework/Core/Http.cs

@@ -13,18 +13,21 @@ namespace Core
 
     internal class Http : IHttp
     {
-        private static readonly object httpLock = new object();
+        private static object lockLock = new object();
+        private static readonly Dictionary<string, object> httpLocks = new Dictionary<string, object>();
         private readonly ILog log;
         private readonly ITimeSet timeSet;
         private readonly Action<HttpClient> onClientCreated;
+        private readonly string id;
 
-        internal Http(ILog log, ITimeSet timeSet)
-            : this(log, timeSet, DoNothing)
+        internal Http(string id, ILog log, ITimeSet timeSet)
+            : this(id, log, timeSet, DoNothing)
         {
         }
 
-        internal Http(ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
+        internal Http(string id, ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
         {
+            this.id = id;
             this.log = log;
             this.timeSet = timeSet;
             this.onClientCreated = onClientCreated;
@@ -63,12 +66,22 @@ namespace Core
 
         private T LockRetry<T>(Func<T> operation, Retry retry)
         {
+            var httpLock = GetLock();
             lock (httpLock)
             {
                 return retry.Run(operation);
             }
         }
 
+        private object GetLock()
+        {
+            lock (lockLock) // I had to.
+            {
+                if (!httpLocks.ContainsKey(id)) httpLocks.Add(id, new object());
+                return httpLocks[id];
+            }
+        }
+
         private HttpClient GetClient()
         {
             var client = new HttpClient();

Framework/Core/PluginTools.cs

@@ -27,9 +27,9 @@ namespace Core
 
     public interface IHttpFactoryTool
     {
-        IHttp CreateHttp(Action<HttpClient> onClientCreated);
-        IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
-        IHttp CreateHttp();
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet timeSet);
+        IHttp CreateHttp(string id);
     }
 
     public interface IFileTool
@@ -58,19 +58,19 @@ namespace Core
             log.Prefix = prefix;
         }
 
-        public IHttp CreateHttp(Action<HttpClient> onClientCreated)
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
         {
-            return CreateHttp(onClientCreated, TimeSet);
+            return CreateHttp(id, onClientCreated, TimeSet);
         }
 
-        public IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet ts)
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet ts)
         {
-            return new Http(log, ts, onClientCreated);
+            return new Http(id, log, ts, onClientCreated);
         }
 
-        public IHttp CreateHttp()
+        public IHttp CreateHttp(string id)
         {
-            return new Http(log, TimeSet);
+            return new Http(id, log, TimeSet);
         }
 
         public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)

Framework/Core/TimeSet.cs

@@ -34,12 +34,12 @@
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromMinutes(3);
+            return TimeSpan.FromMinutes(2);
         }
 
         public TimeSpan HttpRetryTimeout()
         {
-            return TimeSpan.FromMinutes(10);
+            return TimeSpan.FromMinutes(5);
         }
 
         public TimeSpan HttpCallRetryDelay()

Framework/DiscordRewards/DiscordRewards.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -3,12 +3,12 @@
     public class GiveRewardsCommand
     {
         public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
-        public MarketAverage[] Averages { get; set; } = Array.Empty<MarketAverage>();
-        public string[] EventsOverview { get; set; } = Array.Empty<string>();
+        public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty<ChainEventMessage>();
+        public string[] Errors { get; set; } = Array.Empty<string>();
 
         public bool HasAny()
         {
-            return Rewards.Any() || Averages.Any() || EventsOverview.Any();
+            return Rewards.Any() || EventsOverview.Any();
         }
     }
 
@@ -18,14 +18,9 @@
         public string[] UserAddresses { get; set; } = Array.Empty<string>();
     }
 
-    public class MarketAverage
+    public class ChainEventMessage
     {
-        public int NumberOfFinished { get; set; }
-        public int TimeRangeSeconds { get; set; }
-        public float Price { get; set; }
-        public float Size { get; set; }
-        public float Duration { get; set; }
-        public float Collateral { get; set; }
-        public float ProofProbability { get; set; }
+        public ulong BlockNumber { get; set; }
+        public string Message { get; set; } = string.Empty;
     }
 }

Framework/DiscordRewards/RewardRepo.cs

@@ -1,53 +1,53 @@
-using Utils;
-
-namespace DiscordRewards
+namespace DiscordRewards
 {
     public class RewardRepo
     {
         private static string Tag => RewardConfig.UsernameTag;
 
-        public RewardConfig[] Rewards { get; } = new RewardConfig[]
-        {
-            // Filled any slot
-            new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
-            {
-                Type = CheckType.HostFilledSlot
-            }),
+        public RewardConfig[] Rewards { get; } = new RewardConfig[0];
 
-            // Finished any slot
-            new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
-            {
-                Type = CheckType.HostFinishedSlot
-            }),
+        // Example configuration, from test server:
+        //{
+        //    // Filled any slot
+        //    new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFilledSlot
+        //    }),
 
-            // Finished a sizable slot
-            new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig
-            {
-                Type = CheckType.HostFinishedSlot,
-                MinSlotSize = 10.MB(),
-                MinDuration = TimeSpan.FromMinutes(5.0),
-            }),
+        //    // Finished any slot
+        //    new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFinishedSlot
+        //    }),
 
-            // Posted any contract
-            new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
-            {
-                Type = CheckType.ClientPostedContract
-            }),
+        //    // Finished a sizable slot
+        //    new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFinishedSlot,
+        //        MinSlotSize = 10.MB(),
+        //        MinDuration = TimeSpan.FromMinutes(5.0),
+        //    }),
 
-            // Started any contract
-            new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
-            {
-                Type = CheckType.ClientStartedContract
-            }),
+        //    // Posted any contract
+        //    new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientPostedContract
+        //    }),
 
-            // Started a sizable contract
-            new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig
-            {
-                Type = CheckType.ClientStartedContract,
-                MinNumberOfHosts = 4,
-                MinSlotSize = 10.MB(),
-                MinDuration = TimeSpan.FromMinutes(5.0),
-            })
-        };
+        //    // Started any contract
+        //    new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientStartedContract
+        //    }),
+
+        //    // Started a sizable contract
+        //    new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientStartedContract,
+        //        MinNumberOfHosts = 4,
+        //        MinSlotSize = 10.MB(),
+        //        MinDuration = TimeSpan.FromMinutes(5.0),
+        //    })
+        //};
     }
 }

Framework/FileUtils/FileManager.cs

@@ -7,27 +7,35 @@ namespace FileUtils
     {
         TrackedFile CreateEmptyFile(string label = "");
         TrackedFile GenerateFile(ByteSize size, string label = "");
+        TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "");
         void DeleteAllFiles();
         void ScopedFiles(Action action);
         T ScopedFiles<T>(Func<T> action);
     }
 
+    public interface IGenerateOption
+    {
+        IGenerateOption Random(ByteSize size);
+        IGenerateOption StringRepeat(string str, ByteSize size);
+        IGenerateOption StringRepeat(string str, int times);
+        IGenerateOption ByteRepeat(byte[] bytes, ByteSize size);
+        IGenerateOption ByteRepeat(byte[] bytes, int times);
+    }
+
     public class FileManager : IFileManager
     {
-        public const int ChunkSize = 1024 * 1024 * 100;
-        private static NumberSource folderNumberSource = new NumberSource(0);
-        private readonly Random random = new Random();
+        private static readonly NumberSource folderNumberSource = new NumberSource(0);
         private readonly ILog log;
-        private readonly string rootFolder;
         private readonly string folder;
         private readonly List<List<TrackedFile>> fileSetStack = new List<List<TrackedFile>>();
 
+        public const int ChunkSize = 1024 * 1024 * 100;
+
         public FileManager(ILog log, string rootFolder)
         {
             folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
 
             this.log = log;
-            this.rootFolder = rootFolder;
         }
 
         public TrackedFile CreateEmptyFile(string label = "")
@@ -41,10 +49,15 @@ namespace FileUtils
             return result;
         }
 
-        public TrackedFile GenerateFile(ByteSize size, string label)
+        public TrackedFile GenerateFile(ByteSize size, string label = "")
+        {
+            return GenerateFile(o => o.Random(size), label);
+        }
+
+        public TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "")
         {
             var sw = Stopwatch.Begin(log);
-            var result = GenerateRandomFile(size, label);
+            var result = RunGenerators(options, label);
             sw.End($"Generated file {result.Describe()}.");
             return result;
         }
@@ -57,16 +70,27 @@ namespace FileUtils
         public void ScopedFiles(Action action)
         {
             PushFileSet();
-            action();
-            PopFileSet();
+            try
+            {
+                action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
         }
 
         public T ScopedFiles<T>(Func<T> action)
         {
             PushFileSet();
-            var result = action();
-            PopFileSet();
-            return result;
+            try
+            {
+                return action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
         }
 
         private void PushFileSet()
@@ -89,26 +113,35 @@ namespace FileUtils
             if (!Directory.GetFiles(folder).Any()) DeleteDirectory();
         }
 
-        private TrackedFile GenerateRandomFile(ByteSize size, string label)
+        private TrackedFile RunGenerators(Action<IGenerateOption> options, string label)
         {
             var result = CreateEmptyFile(label);
-            CheckSpaceAvailable(result, size);
+            var generators = GetGenerators(options);
+            CheckSpaceAvailable(result, generators.GetRequiredSpace());
 
-            GenerateFileBytes(result, size);
+            using var stream = new FileStream(result.Filename, FileMode.Append);
+            generators.Run(stream);
             return result;
         }
 
-        private void CheckSpaceAvailable(TrackedFile testFile, ByteSize size)
+        private GeneratorCollection GetGenerators(Action<IGenerateOption> options)
+        {
+            var result = new GeneratorCollection();
+            options(result);
+            return result;
+        }
+
+        private void CheckSpaceAvailable(TrackedFile testFile, long requiredSize)
         {
             var file = new FileInfo(testFile.Filename);
             var drive = new DriveInfo(file.Directory!.Root.FullName);
 
             var spaceAvailable = drive.TotalFreeSpace;
 
-            if (spaceAvailable < size.SizeInBytes)
+            if (spaceAvailable < requiredSize)
             {
                 var msg = $"Not enough disk space. " +
-                    $"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
+                    $"{Formatter.FormatByteSize(requiredSize)} required. " +
                     $"{Formatter.FormatByteSize(spaceAvailable)} available.";
 
                 log.Log(msg);
@@ -116,34 +149,6 @@ namespace FileUtils
             }
         }
 
-        private void GenerateFileBytes(TrackedFile result, ByteSize size)
-        {
-            long bytesLeft = size.SizeInBytes;
-            int chunkSize = ChunkSize;
-            while (bytesLeft > 0)
-            {
-                try
-                {
-                    var length = Math.Min(bytesLeft, chunkSize);
-                    AppendRandomBytesToFile(result, length);
-                    bytesLeft -= length;
-                }
-                catch
-                {
-                    chunkSize = chunkSize / 2;
-                    if (chunkSize < 1024) throw;
-                }
-            }
-        }
-
-        private void AppendRandomBytesToFile(TrackedFile result, long length)
-        {
-            var bytes = new byte[length];
-            random.NextBytes(bytes);
-            using var stream = new FileStream(result.Filename, FileMode.Append);
-            stream.Write(bytes, 0, bytes.Length);
-        }
-
         private void EnsureDirectory()
         {
             Directory.CreateDirectory(folder);

Framework/FileUtils/FileUtils.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/FileUtils/Generators.cs

@@ -0,0 +1,141 @@
+using System.Text;
+using Utils;
+
+namespace FileUtils
+{
+    public class GeneratorCollection : IGenerateOption
+    {
+        private readonly List<IGenerator> generators = new List<IGenerator>();
+
+        public IGenerateOption ByteRepeat(byte[] bytes, ByteSize size)
+        {
+            var times = size.SizeInBytes / bytes.Length;
+            generators.Add(new ByteRepeater(bytes, times));
+            return this;
+        }
+
+        public IGenerateOption ByteRepeat(byte[] bytes, int times)
+        {
+            generators.Add(new ByteRepeater(bytes, times));
+            return this;
+        }
+
+        public IGenerateOption Random(ByteSize size)
+        {
+            generators.Add(new RandomGenerator(size));
+            return this;
+        }
+
+        public IGenerateOption StringRepeat(string str, ByteSize size)
+        {
+            var times = size.SizeInBytes / str.Length;
+            generators.Add(new StringRepeater(str, times));
+            return this;
+        }
+
+        public IGenerateOption StringRepeat(string str, int times)
+        {
+            generators.Add(new StringRepeater(str, times));
+            return this;
+        }
+
+        public void Run(FileStream file)
+        {
+            foreach (var generator in generators)
+            {
+                generator.Generate(file);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return generators.Sum(g => g.GetRequiredSpace());
+        }
+    }
+
+    public interface IGenerator
+    {
+        void Generate(FileStream file);
+        long GetRequiredSpace();
+    }
+
+    public class ByteRepeater : IGenerator
+    {
+        private readonly byte[] bytes;
+        private readonly long times;
+
+        public ByteRepeater(byte[] bytes, long times)
+        {
+            this.bytes = bytes;
+            this.times = times;
+        }
+
+        public void Generate(FileStream file)
+        {
+            for (var i = 0; i < times; i++)
+            {
+                file.Write(bytes, 0, bytes.Length);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return bytes.Length * times;
+        }
+    }
+
+    public class StringRepeater : IGenerator
+    {
+        private readonly string str;
+        private readonly long times;
+
+        public StringRepeater(string str, long times)
+        {
+            this.str = str;
+            this.times = times;
+        }
+
+        public void Generate(FileStream file)
+        {
+            using var writer = new StreamWriter(file);
+            for (var i = 0; i < times; i++)
+            {
+                writer.Write(str);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return Encoding.ASCII.GetBytes(str).Length * times;
+        }
+    }
+
+    public class RandomGenerator : IGenerator
+    {
+        private readonly Random random = new Random();
+        private readonly ByteSize size;
+
+        public RandomGenerator(ByteSize size)
+        {
+            this.size = size;
+        }
+
+        public void Generate(FileStream file)
+        {
+            var bytesLeft = size.SizeInBytes;
+            while (bytesLeft > 0)
+            {
+                var size = Math.Min(bytesLeft, FileManager.ChunkSize);
+                var bytes = new byte[size];
+                random.NextBytes(bytes);
+                file.Write(bytes, 0, bytes.Length);
+                bytesLeft -= size;
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return size.SizeInBytes;
+        }
+    }
+}

Framework/GethConnector/GethConnector.cs

@@ -1,4 +1,6 @@
-using CodexContractsPlugin;
+using BlockchainUtils;
+using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 
@@ -18,18 +20,29 @@ namespace GethConnector
                 return null;
             }
 
+            var gethNode = new CustomGethNode(log, new BlockCache(), GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+
+            var config = GetCodexMarketplaceConfig(gethNode, GethInput.MarketplaceAddress);
+
             var contractsDeployment = new CodexContractsDeployment(
+                config: config,
                 marketplaceAddress: GethInput.MarketplaceAddress,
                 abi: GethInput.ABI,
                 tokenAddress: GethInput.TokenAddress
             );
 
-            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
             var contracts = new CodexContractsAccess(log, gethNode, contractsDeployment);
 
             return new GethConnector(gethNode, contracts);
         }
 
+        private static MarketplaceConfig GetCodexMarketplaceConfig(IGethNode gethNode, string marketplaceAddress)
+        {
+            var func = new ConfigurationFunctionBase();
+            var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
+            return response.ReturnValue1;
+        }
+
         private GethConnector(IGethNode gethNode, ICodexContracts codexContracts)
         {
             GethNode = gethNode;

Framework/GethConnector/GethConnector.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -91,7 +91,7 @@ namespace KubernetesWorkflow
         private void DownloadCrashedContainerLogs(Kubernetes client)
         {
             using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
-            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
+            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName, containerName);
             handler.Log(stream);
         }
     }

Framework/KubernetesWorkflow/DownloadedLog.cs

@@ -1,13 +1,16 @@
-using KubernetesWorkflow;
-using Logging;
+using Logging;
 
-namespace Core
+namespace KubernetesWorkflow
 {
     public interface IDownloadedLog
     {
+        string ContainerName { get; }
+
         void IterateLines(Action<string> action);
+        void IterateLines(Action<string> action, params string[] thatContain);
         string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
+        string GetFilepath();
         void DeleteFile();
     }
 
@@ -15,11 +18,14 @@ namespace Core
     {
         private readonly LogFile logFile;
 
-        internal DownloadedLog(WriteToFileLogHandler logHandler)
+        internal DownloadedLog(WriteToFileLogHandler logHandler, string containerName)
         {
             logFile = logHandler.LogFile;
+            ContainerName = containerName;
         }
 
+        public string ContainerName { get; }
+
         public void IterateLines(Action<string> action)
         {
             using var file = File.OpenRead(logFile.FullFilename);
@@ -33,45 +39,34 @@ namespace Core
             }
         }
 
+        public void IterateLines(Action<string> action, params string[] thatContain)
+        {
+            IterateLines(line =>
+            {
+                if (thatContain.All(line.Contains))
+                {
+                    action(line);
+                }
+            });
+        }
+
         public string[] GetLinesContaining(string expectedString)
         {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-            var lines = new List<string>();
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (line.Contains(expectedString))
-                {
-                    lines.Add(line);
-                }
-                line = streamReader.ReadLine();
-            }
-
-            return lines.ToArray(); ;
+            return FindLinesThatContain([expectedString]);
         }
 
         public string[] FindLinesThatContain(params string[] tags)
         {
             var result = new List<string>();
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (tags.All(line.Contains))
-                {
-                    result.Add(line);
-                }
-
-                line = streamReader.ReadLine();
-            }
-
+            IterateLines(result.Add, tags);
             return result.ToArray();
         }
 
+        public string GetFilepath()
+        {
+            return logFile.FullFilename;
+        }
+
         public void DeleteFile()
         {
             File.Delete(logFile.FullFilename);

Framework/KubernetesWorkflow/K8sController.cs

@@ -906,7 +906,7 @@ namespace KubernetesWorkflow
                 var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
                 log.Error(msg);
 
-                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg, deploymentName), tailLines: null, previous: true);
 
                 throw new Exception(msg);
             }

Framework/KubernetesWorkflow/K8sNameUtils.cs

@@ -24,8 +24,8 @@
                 .Replace("]", "-")
                 .Replace(",", "-");
 
-            result = result.Trim('-');
             if (result.Length > maxLength) result = result.Substring(0, maxLength);
+            result = result.Trim('-');
 
             return result;
         }

Framework/KubernetesWorkflow/KubernetesWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>KubernetesWorkflow</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>

Framework/KubernetesWorkflow/LogHandler.cs

@@ -25,9 +25,9 @@ namespace KubernetesWorkflow
 
     public class WriteToFileLogHandler : LogHandler, ILogHandler
     {
-        public WriteToFileLogHandler(ILog sourceLog, string description)
+        public WriteToFileLogHandler(ILog sourceLog, string description, string addFileName)
         {
-            LogFile = sourceLog.CreateSubfile();
+            LogFile = sourceLog.CreateSubfile(addFileName);
 
             var msg = $"{description} -->> {LogFile.FullFilename}";
             sourceLog.Log(msg);

Framework/KubernetesWorkflow/Recipe/ContainerAdditionals.cs

@@ -1,4 +1,5 @@
-using Newtonsoft.Json.Linq;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 
 namespace KubernetesWorkflow.Recipe
 {
@@ -21,14 +22,13 @@ namespace KubernetesWorkflow.Recipe
             var typeName = GetTypeName(typeof(T));
             var userData = Additionals.SingleOrDefault(a => a.Type == typeName);
             if (userData == null) return default;
-            var jobject = (JObject)userData.UserData;
-            return jobject.ToObject<T>();
+            return JsonConvert.DeserializeObject<T>(userData.UserData);
         }
 
         private static Additional ConvertToAdditional(object userData)
         {
             var typeName = GetTypeName(userData.GetType());
-            return new Additional(typeName, userData);
+            return new Additional(typeName, JsonConvert.SerializeObject(userData));
         }
 
         private static string GetTypeName(Type type)
@@ -41,13 +41,13 @@ namespace KubernetesWorkflow.Recipe
 
     public class Additional
     {
-        public Additional(string type, object userData)
+        public Additional(string type, string userData)
         {
             Type = type;
             UserData = userData;
         }
 
         public string Type { get; }
-        public object UserData { get; }
+        public string UserData { get; }
     }
 }

Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs

@@ -2,8 +2,9 @@
 {
     public class ContainerRecipe
     {
-        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, CommandOverride commandOverride, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
+        public ContainerRecipe(DateTime recipeCreatedUtc, int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, CommandOverride commandOverride, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
         {
+            RecipeCreatedUtc = recipeCreatedUtc;
             Number = number;
             NameOverride = nameOverride;
             Image = image;
@@ -31,6 +32,7 @@
             if (exposedPorts.Any(p => string.IsNullOrEmpty(p.Tag))) throw new Exception("Port tags are required for all exposed ports.");
         }
 
+        public DateTime RecipeCreatedUtc { get; }
         public string Name { get; }
         public int Number { get; }
         public string? NameOverride { get; }

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -25,7 +25,7 @@ namespace KubernetesWorkflow.Recipe
 
             Initialize(config);
 
-            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, commandOverride, setCriticalPriority,
+            var recipe = new ContainerRecipe(DateTime.UtcNow, containerNumber, config.NameOverride, Image, resources, schedulingAffinity, commandOverride, setCriticalPriority,
                 exposedPorts.ToArray(),
                 internalPorts.ToArray(),
                 envVars.ToArray(),

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -16,6 +16,7 @@ namespace KubernetesWorkflow
         CrashWatcher CreateCrashWatcher(RunningContainer container);
         void Stop(RunningPod pod, bool waitTillStopped);
         void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
+        IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
         void DeleteNamespace(bool wait);
         void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
@@ -60,7 +61,7 @@ namespace KubernetesWorkflow
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
-                var rc = new RunningPod(startupConfig, startResult, containers);
+                var rc = new RunningPod(Guid.NewGuid().ToString(), startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
 
                 if (startResult.ExternalService != null)
@@ -99,11 +100,19 @@ namespace KubernetesWorkflow
 
         public void Stop(RunningPod runningPod, bool waitTillStopped)
         {
+            if (runningPod.IsStopped) return;
+            foreach (var c in runningPod.Containers)
+            {
+                c.StopLog = DownloadContainerLog(c);
+            }
+            runningPod.IsStopped = true;
+
             K8s(controller =>
             {
                 controller.Stop(runningPod.StartResult, waitTillStopped);
-                cluster.Configuration.Hooks.OnContainersStopped(runningPod);
             });
+
+            cluster.Configuration.Hooks.OnContainersStopped(runningPod);
         }
 
         public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null)
@@ -114,6 +123,20 @@ namespace KubernetesWorkflow
             });
         }
 
+        public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null)
+        {
+            var msg = $"Downloading container log for '{container.Name}'";
+            log.Log(msg);
+            var logHandler = new WriteToFileLogHandler(log, msg, container.Name);
+
+            K8s(controller =>
+            {
+                controller.DownloadPodLog(container, logHandler, tailLines, previous);
+            });
+
+            return new DownloadedLog(logHandler, container.Name);
+        }
+
         public string ExecuteCommand(RunningContainer container, string command, params string[] args)
         {
             return K8s(controller =>
@@ -147,7 +170,7 @@ namespace KubernetesWorkflow
                 var addresses = CreateContainerAddresses(startResult, r);
                 log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}");
 
-                return new RunningContainer(name, r, addresses);
+                return new RunningContainer(Guid.NewGuid().ToString(), name, r, addresses);
 
             }).ToArray();
         }

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -7,27 +7,29 @@ namespace KubernetesWorkflow.Types
 {
     public class RunningContainer
     {
-        public RunningContainer(string name, ContainerRecipe recipe, ContainerAddress[] addresses)
+        public RunningContainer(string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses)
         {
+            Id = id;
             Name = name;
             Recipe = recipe;
             Addresses = addresses;
         }
 
+        public string Id { get; }
         public string Name { get; }
         public ContainerRecipe Recipe { get; }
         public ContainerAddress[] Addresses { get; }
+        public IDownloadedLog? StopLog { get; internal set; }
 
         [JsonIgnore]
         public RunningPod RunningPod { get; internal set; } = null!;
 
-        public Address GetAddress(ILog log, string portTag)
+        public Address GetAddress(string portTag)
         {
             var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
             if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);
 
             var select = SelectAddress(addresses);
-            log.Debug($"Container '{Name}' selected for tag '{portTag}' address: '{select}'");
             return select.Address;
         }
 
@@ -50,5 +52,21 @@ namespace KubernetesWorkflow.Types
             }
             throw new Exception("Running location not known.");
         }
+
+        public override string ToString()
+        {
+            return Name;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is RunningContainer container &&
+                   Id == container.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
     }
 }

Framework/KubernetesWorkflow/Types/RunningPod.cs

@@ -4,8 +4,9 @@ namespace KubernetesWorkflow.Types
 {
     public class RunningPod
     {
-        public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        public RunningPod(string id, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
         {
+            Id = id;
             StartupConfig = startupConfig;
             StartResult = startResult;
             Containers = containers;
@@ -13,6 +14,7 @@ namespace KubernetesWorkflow.Types
             foreach (var c in containers) c.RunningPod = this;
         }
 
+        public string Id { get; }
         public StartupConfig StartupConfig { get; }
         public StartResult StartResult { get; }
         public RunningContainer[] Containers { get; }
@@ -23,10 +25,30 @@ namespace KubernetesWorkflow.Types
             get { return $"'{string.Join("&", Containers.Select(c => c.Name).ToArray())}'"; }
         }
 
+        [JsonIgnore]
+        public bool IsStopped { get; internal set; }
+
         public string Describe()
         {
             return string.Join(",", Containers.Select(c => c.Name));
         }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is RunningPod pod &&
+                   Id == pod.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
+
+        public override string ToString()
+        {
+            if (IsStopped) return Name + " (*)";
+            return Name;
+        }
     }
 
     public static class RunningContainersExtensions

Framework/Logging/BaseLog.cs

@@ -8,7 +8,7 @@ namespace Logging
         void Debug(string message = "", int skipFrames = 0);
         void Error(string message);
         void AddStringReplace(string from, string to);
-        LogFile CreateSubfile(string ext = "log");
+        LogFile CreateSubfile(string addName, string ext = "log");
     }
 
     public abstract class BaseLog : ILog
@@ -72,12 +72,16 @@ namespace Logging
             File.Delete(LogFile.FullFilename);
         }
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
+            addName = addName
+                .Replace("<", "")
+                .Replace(">", "");
+
+            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}_{addName}", ext);
         }
 
-        private string ApplyReplacements(string str)
+        protected string ApplyReplacements(string str)
         {
             if (IsDebug) return str;
             foreach (var replacement in replacements)

Framework/Logging/ConsoleLog.cs

@@ -9,7 +9,7 @@
 
         public override void Log(string message)
         {
-            Console.WriteLine(message);
+            Console.WriteLine(ApplyReplacements(message));
         }
     }
 }

Framework/Logging/LogFile.cs

@@ -1,4 +1,6 @@
-namespace Logging
+using Utils;
+
+namespace Logging
 {
     public class LogFile
     {
@@ -49,7 +51,7 @@
 
         private static string GetTimestamp()
         {
-            return $"[{DateTime.UtcNow.ToString("o")}]";
+            return $"[{Time.FormatTimestamp(DateTime.UtcNow)}]";
         }
 
         private void EnsurePathExists(string filename)

Framework/Logging/LogPrefixer.cs

@@ -18,9 +18,9 @@
         public string Prefix { get; set; } = string.Empty;
 
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return backingLog.CreateSubfile(ext);
+            return backingLog.CreateSubfile(addName, ext);
         }
 
         public void Debug(string message = "", int skipFrames = 0)

Framework/Logging/LogSplitter.cs

@@ -14,9 +14,9 @@
             OnAll(l => l.AddStringReplace(from, to));
         }
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return targetLogs.First().CreateSubfile(ext);
+            return targetLogs.First().CreateSubfile(addName, ext);
         }
 
         public void Debug(string message = "", int skipFrames = 0)

Framework/Logging/Logging.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Logging</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -1,25 +1,25 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace NethereumWorkflow
 {
     public class NethereumInteraction
     {
-        // BlockCache is a static instance: It stays alive for the duration of the application runtime.
-        private readonly static BlockCache blockCache = new BlockCache();
+        private readonly BlockCache blockCache;
 
         private readonly ILog log;
         private readonly Web3 web3;
 
-        internal NethereumInteraction(ILog log, Web3 web3)
+        internal NethereumInteraction(ILog log, Web3 web3, BlockCache blockCache)
         {
             this.log = log;
             this.web3 = web3;
+            this.blockCache = blockCache;
         }
 
         public string SendEth(string toAddress, decimal ethAmount)

Framework/NethereumWorkflow/NethereumInteractionCreator.cs

@@ -1,4 +1,5 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.Web3;
 
 namespace NethereumWorkflow
@@ -6,13 +7,15 @@ namespace NethereumWorkflow
     public class NethereumInteractionCreator
     {
         private readonly ILog log;
+        private readonly BlockCache blockCache;
         private readonly string ip;
         private readonly int port;
         private readonly string privateKey;
 
-        public NethereumInteractionCreator(ILog log, string ip, int port, string privateKey)
+        public NethereumInteractionCreator(ILog log, BlockCache blockCache, string ip, int port, string privateKey)
         {
             this.log = log;
+            this.blockCache = blockCache;
             this.ip = ip;
             this.port = port;
             this.privateKey = privateKey;
@@ -21,7 +24,7 @@ namespace NethereumWorkflow
         public NethereumInteraction CreateWorkflow()
         {
             log.Debug("Starting interaction to " + ip + ":" + port);
-            return new NethereumInteraction(log, CreateWeb3());
+            return new NethereumInteraction(log, CreateWeb3(), blockCache);
         }
 
         private Web3 CreateWeb3()

Framework/NethereumWorkflow/NethereumWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	<RootNamespace>NethereumWorkflow</RootNamespace>
 	<ImplicitUsings>enable</ImplicitUsings>
 	<Nullable>enable</Nullable>
@@ -12,6 +12,7 @@
 	</ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\BlockchainUtils\BlockchainUtils.csproj" />
     <ProjectReference Include="..\Logging\Logging.csproj" />
     <ProjectReference Include="..\Utils\Utils.csproj" />
   </ItemGroup>

Framework/NethereumWorkflow/Web3Wrapper.cs

@@ -1,16 +1,11 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
 using Utils;
 
 namespace NethereumWorkflow
 {
-    public interface IWeb3Blocks
-    {
-        ulong GetCurrentBlockNumber();
-        DateTime? GetTimestampForBlock(ulong blockNumber);
-    }
-
     public class Web3Wrapper : IWeb3Blocks
     {
         private readonly Web3 web3;

Framework/OverwatchTranscript/ActionQueue.cs

@@ -0,0 +1,61 @@
+namespace OverwatchTranscript
+{
+    public class ActionQueue
+    {
+        // Using ConcurrentQueue<> here would make this process slower.
+        private readonly object queueLock = new object();
+        private readonly AutoResetEvent signal = new AutoResetEvent(false);
+        private List<Action> queue = new List<Action>();
+        private Task queueWorker = null!;
+        private bool stopping = false;
+
+        public void Start()
+        {
+            queueWorker = Task.Run(QueueWorker);
+        }
+
+        public int Count { get; private set; }
+
+        public void StopAndJoin()
+        {
+            stopping = true;
+            queueWorker.Wait();
+            if (queue.Count > 0) throw new Exception("not all acions handled");
+            queueWorker.Dispose();
+        }
+
+        public void Add(Action action)
+        {
+            if (stopping) throw new Exception("queue stopping");
+
+            lock (queueLock)
+            {
+                queue.Add(action);
+                Count = queue.Count;
+            }
+            signal.Set();
+        }
+
+        private void QueueWorker()
+        {
+            while (true)
+            {
+                signal.WaitOne(10);
+
+                List<Action> work = null!;
+                lock (queueLock)
+                {
+                    work = queue;
+                    queue = new List<Action>();
+                    Count = 0;
+                }
+                if (stopping && !work.Any()) return;
+                
+                foreach (var action in work)
+                {
+                    action();
+                }
+            }
+        }
+    }
+}

Framework/OverwatchTranscript/BucketSet.cs

@@ -0,0 +1,110 @@
+using Logging;
+using System.Collections.Concurrent;
+
+namespace OverwatchTranscript
+{
+    public class BucketSet
+    {
+        private const int numberOfActiveBuckets = 10;
+        private readonly ILog log;
+        private readonly string workingDir;
+        private readonly object _bucketLock = new object();
+        private readonly List<EventBucketWriter> fullBuckets = new List<EventBucketWriter>();
+        private readonly List<EventBucketWriter> activeBuckets = new List<EventBucketWriter>();
+        private readonly ActionQueue queue = new ActionQueue();
+        private int activeBucketIndex = 0;
+        private bool closed = false;
+        private string internalErrors = string.Empty;
+        
+        public BucketSet(ILog log, string workingDir)
+        {
+            this.log = log;
+            this.workingDir = workingDir;
+
+            for (var i = 0; i < numberOfActiveBuckets;i++)
+            {
+                AddNewBucket();
+            }
+
+            queue.Start();
+        }
+
+        public void Add(DateTime utc, object payload)
+        {
+            if (closed) throw new Exception("Buckets already closed!");
+            queue.Add(() => AddInternal(utc, payload));
+            
+            if (queue.Count > 1000)
+            {
+                Thread.Sleep(1);
+            }
+        }
+
+        public IFinalizedBucket[] FinalizeBuckets()
+        {
+            closed = true;
+            queue.StopAndJoin();
+
+            if (IsEmpty()) throw new Exception("No entries have been added.");
+            if (!string.IsNullOrEmpty(internalErrors)) throw new Exception(internalErrors);
+
+            var buckets = fullBuckets.Concat(activeBuckets).ToArray();
+            log.Debug($"Finalizing {buckets.Length} buckets...");
+
+            var finalized = new ConcurrentBag<IFinalizedBucket>();
+            var tasks = Parallel.ForEach(buckets, b => finalized.Add(b.FinalizeBucket()));
+            if (!tasks.IsCompleted) throw new Exception("Failed to finalize buckets: " + tasks);
+
+            return finalized.ToArray();
+        }
+
+        private bool IsEmpty()
+        {
+            return fullBuckets.All(b => b.Count == 0) && activeBuckets.All(b => b.Count == 0);
+        }
+
+        private void AddInternal(DateTime utc, object payload)
+        {
+            try
+            {
+                lock (_bucketLock)
+                {
+                    var current = activeBuckets[activeBucketIndex];
+                    current.Add(utc, payload);
+                    activeBucketIndex = (activeBucketIndex + 1) % numberOfActiveBuckets;
+
+                    if (current.IsFull)
+                    {
+                        log.Debug("Bucket is full. New bucket...");
+                        fullBuckets.Add(current);
+                        activeBuckets.Remove(current);
+                        AddNewBucket();
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                internalErrors += ex.ToString();
+                log.Error(ex.ToString());
+            }
+        }
+
+        private static int bucketSizeIndex = 0;
+        private static int[] bucketSizes = new[]
+        {
+            10000,
+            15000,
+            20000,
+        };
+
+        private void AddNewBucket()
+        {
+            lock (_bucketLock)
+            {
+                var size = bucketSizes[bucketSizeIndex];
+                bucketSizeIndex = (bucketSizeIndex + 1) % bucketSizes.Length;
+                activeBuckets.Add(new EventBucketWriter(log, Path.Combine(workingDir, Guid.NewGuid().ToString()), size));
+            }
+        }
+    }
+}

Framework/OverwatchTranscript/EventBucketReader.cs

@@ -0,0 +1,151 @@
+using Logging;
+using Newtonsoft.Json;
+using System.Collections.Concurrent;
+
+namespace OverwatchTranscript
+{
+    public interface IFinalizedBucket
+    {
+        bool IsEmpty { get; }
+        void Update();
+        DateTime? SeeTopUtc();
+        BucketTop? TakeTop();
+    }
+
+    public class BucketTop
+    {
+        public BucketTop(DateTime utc, OverwatchEvent[] events)
+        {
+            Utc = utc;
+            Events = events;
+        }
+
+        public DateTime Utc { get; }
+        public OverwatchEvent[] Events { get; }
+    }
+
+    public class EventBucketReader : IFinalizedBucket
+    {
+        private readonly string bucketFile;
+        private readonly ConcurrentQueue<BucketTop> topQueue = new ConcurrentQueue<BucketTop>();
+        private readonly AutoResetEvent itemDequeued = new AutoResetEvent(false);
+        private readonly AutoResetEvent itemEnqueued = new AutoResetEvent(false);
+        private bool sourceIsEmpty;
+
+        public EventBucketReader(ILog log, string bucketFile)
+        {
+            this.bucketFile = bucketFile;
+            if (!File.Exists(bucketFile)) throw new Exception("Doesn't exist: " + bucketFile);
+
+            log.Debug("Read Bucket open: " + bucketFile);
+
+            Task.Run(ReadBucket);
+        }
+        
+        public bool IsEmpty { get; private set; }
+
+        public void Update()
+        {
+            if (IsEmpty) return;
+            while (topQueue.Count == 0)
+            {
+                UpdateIsEmpty();
+                if (IsEmpty) return;
+
+                itemDequeued.Set();
+                itemEnqueued.WaitOne(200);
+            }
+        }
+
+        public DateTime? SeeTopUtc()
+        {
+            if (IsEmpty) return null;
+            if (topQueue.TryPeek(out BucketTop? top))
+            {
+                return top.Utc;
+            }
+            return null;
+        }
+
+        public BucketTop? TakeTop()
+        {
+            if (IsEmpty) return null;
+            if (topQueue.TryDequeue(out BucketTop? top))
+            {
+                itemDequeued.Set();
+                return top;
+            }
+            return null;
+        }
+
+        private void ReadBucket()
+        {
+            using var file = File.OpenRead(bucketFile);
+            using var reader = new StreamReader(file);
+
+            while (true)
+            {
+                while (topQueue.Count < 5)
+                {
+                    var top = CreateNewTop(reader);
+                    if (top != null)
+                    {
+                        topQueue.Enqueue(top);
+                        itemEnqueued.Set();
+                    }
+                    else
+                    {
+                        sourceIsEmpty = true;
+                        UpdateIsEmpty();
+                        return;
+                    }
+                }
+
+                itemDequeued.Reset();
+                itemDequeued.WaitOne(5000);
+            }
+        }
+
+        private void UpdateIsEmpty()
+        {
+            var allEmpty = sourceIsEmpty && topQueue.IsEmpty;
+            if (!IsEmpty && allEmpty)
+            {
+                File.Delete(bucketFile);
+                IsEmpty = true;
+            }
+        }
+
+        private EventBucketEntry? nextEntry = null;
+        private BucketTop? CreateNewTop(StreamReader reader)
+        {
+            if (nextEntry == null)
+            {
+                nextEntry = ReadEntry(reader);
+                if (nextEntry == null) return null;
+            }
+
+            var topEntry = nextEntry;
+            var entries = new List<EventBucketEntry>
+            {
+                topEntry
+            };
+
+            nextEntry = ReadEntry(reader);
+            while (nextEntry != null && nextEntry.Utc == topEntry.Utc)
+            {
+                entries.Add(nextEntry);
+                nextEntry = ReadEntry(reader);
+            }
+
+            return new BucketTop(topEntry.Utc, entries.Select(e => e.Event).ToArray());
+        }
+
+        private EventBucketEntry? ReadEntry(StreamReader reader)
+        {
+            var line = reader.ReadLine();
+            if (string.IsNullOrEmpty(line)) return null;
+            return JsonConvert.DeserializeObject<EventBucketEntry>(line);
+        }
+    }
+}

Framework/OverwatchTranscript/EventBucketWriter.cs

@@ -0,0 +1,113 @@
+using Logging;
+using Newtonsoft.Json;
+
+namespace OverwatchTranscript
+{
+    public class EventBucketWriter
+    {
+        private const int MaxBuffer = 1000;
+
+        private readonly object _lock = new object();
+        private bool closed = false;
+        private readonly ILog log;
+        private readonly string bucketFile;
+        private readonly int maxCount;
+        private readonly List<EventBucketEntry> buffer = new List<EventBucketEntry>();
+
+        public EventBucketWriter(ILog log, string bucketFile, int maxCount)
+        {
+            this.log = log;
+            this.bucketFile = bucketFile;
+            this.maxCount = maxCount;
+            if (File.Exists(bucketFile)) throw new Exception("Already exists");
+
+            log.Debug("Write Bucket open: " + bucketFile);
+        }
+
+        public int Count { get; private set; }
+        public bool IsFull { get; private set; }
+
+        public void Add(DateTime utc, object payload)
+        {
+            lock (_lock)
+            {
+                if (closed) throw new Exception("Already closed");
+                AddToBuffer(utc, payload);
+                BufferToFile(emptyBuffer: false);
+            }
+        }
+
+        public IFinalizedBucket FinalizeBucket()
+        {
+            lock (_lock)
+            {
+                closed = true;
+                BufferToFile(emptyBuffer: true);
+                SortFileByTimestamps();
+            }
+            log.Debug($"Finalized bucket with {Count} entries");
+            return new EventBucketReader(log, bucketFile);
+        }
+
+        public override string ToString()
+        {
+            return $"EventBucket: " + Count;
+        }
+
+        private void AddToBuffer(DateTime utc, object payload)
+        {  
+            var typeName = payload.GetType().FullName;
+            if (string.IsNullOrEmpty(typeName)) throw new Exception("Empty typename for payload");
+            if (utc == default) throw new Exception("DateTimeUtc not set");
+
+            var entry = new EventBucketEntry
+            {
+                Utc = utc,
+                Event = new OverwatchEvent
+                {
+                    Type = typeName,
+                    Payload = Json.Serialize(payload)
+                }
+            };
+
+            Count++;
+            IsFull = Count > maxCount;
+
+            buffer.Add(entry);
+        }
+
+        private void BufferToFile(bool emptyBuffer)
+        {
+            if (emptyBuffer || buffer.Count > MaxBuffer)
+            {
+                using var file = File.Open(bucketFile, FileMode.Append);
+                using var writer = new StreamWriter(file);
+                foreach (var entry in buffer)
+                {
+                    writer.WriteLine(Json.Serialize(entry));
+                }
+                log.Debug($"Bucket wrote {buffer.Count} entries to file.");
+                buffer.Clear();
+            }
+        }
+
+        private void SortFileByTimestamps()
+        {
+            var lines = File.ReadAllLines(bucketFile);
+            var entries = lines.Select(Json.Deserialize<EventBucketEntry>)
+                .Cast<EventBucketEntry>()
+                .OrderBy(e => e.Utc)
+                .ToArray();
+
+            File.Delete(bucketFile);
+            File.WriteAllLines(bucketFile, entries.Select(e => Json.Serialize(e)));
+        }
+    }
+
+    [Serializable]
+    public class EventBucketEntry
+    {
+        public DateTime Utc { get; set; }
+        public OverwatchEvent Event { get; set; } = new();
+    }
+}

Framework/OverwatchTranscript/Json.cs

@@ -0,0 +1,27 @@
+using Newtonsoft.Json;
+using System.Globalization;
+
+namespace OverwatchTranscript
+{
+    public static class Json
+    {
+        private static JsonSerializerSettings settings = new JsonSerializerSettings
+        {
+            Formatting = Formatting.None,
+            NullValueHandling = NullValueHandling.Ignore,
+            Culture = CultureInfo.InvariantCulture,
+            DateFormatHandling = DateFormatHandling.IsoDateFormat,
+            FloatFormatHandling = FloatFormatHandling.Symbol
+        };
+
+        public static string Serialize(object obj, Formatting formatting = Formatting.None)
+        {
+            return JsonConvert.SerializeObject(obj, formatting, settings);
+        }
+
+        public static T Deserialize<T>(string json)
+        {
+            return JsonConvert.DeserializeObject<T>(json)!;
+        }
+    }
+}

Framework/OverwatchTranscript/Model.cs

@@ -0,0 +1,56 @@
+namespace OverwatchTranscript
+{
+    [Serializable]
+    public class OverwatchTranscript
+    {
+        public OverwatchHeader Header { get; set; } = new();
+        public OverwatchMomentReference[] MomentReferences { get; set; } = Array.Empty<OverwatchMomentReference>();
+    }
+
+    [Serializable]
+    public class OverwatchMomentReference
+    {
+        public string MomentsFile { get; set; } = string.Empty;
+        public int NumberOfMoments { get; set; }
+        public int NumberOfEvents { get; set; }
+        public DateTime EarliestUtc { get; set; }
+        public DateTime LatestUtc { get; set; }
+    }
+
+    [Serializable]
+    public class OverwatchHeader
+    {
+        public OverwatchCommonHeader Common { get; set; } = new();
+        public OverwatchHeaderEntry[] Entries { get; set; } = Array.Empty<OverwatchHeaderEntry>();
+    }
+
+    [Serializable]
+    public class OverwatchCommonHeader
+    {
+        public long NumberOfMoments { get; set; }
+        public long NumberOfEvents { get; set; }
+        public DateTime EarliestUtc { get; set; }
+        public DateTime LatestUtc { get; set; }
+    }
+
+    [Serializable]
+    public class OverwatchHeaderEntry
+    {
+        public string Key { get; set; } = string.Empty;
+        public string Value { get; set; } = string.Empty;
+    }
+
+    [Serializable]
+    public class OverwatchMoment
+    {
+        public DateTime Utc { get; set; }
+        public OverwatchEvent[] Events { get; set; } = Array.Empty<OverwatchEvent>();
+    }
+
+    [Serializable]
+    public class OverwatchEvent
+    {
+        public string Type { get; set; } = string.Empty;
+        public string Payload { get; set; } = string.Empty;
+    }
+}

Framework/OverwatchTranscript/MomentReader.cs

@@ -0,0 +1,104 @@
+using Newtonsoft.Json;
+
+namespace OverwatchTranscript
+{
+    public class MomentReader
+    {
+        private readonly OverwatchTranscript model;
+        private readonly string workingDir;
+        private int referenceIndex = 0;
+        private int momentsRead = 0;
+        private OpenReference currentRef;
+
+        public MomentReader(OverwatchTranscript model, string workingDir)
+        {
+            this.model = model;
+            this.workingDir = workingDir;
+
+            currentRef = CreateOpenReference();
+        }
+
+        public OverwatchMoment? Next()
+        {
+            if (referenceIndex >= model.MomentReferences.Length) return null;
+
+            var moment = currentRef.ReadNext();
+            if (moment == null)
+            {
+                Close();
+
+                // This reference file ran out.
+                // The number of moments read should match exactly the number of moments
+                // describe in the reference. If not, error:
+                var expected = model.MomentReferences[referenceIndex].NumberOfMoments;
+                if (momentsRead != expected)
+                {
+                    throw new Exception("Number of moments read from referenced file does not match number of moments value in model. " +
+                        $"Reads: { momentsRead} - model.MomentReferences[{referenceIndex}].NumberOfMoment: {expected}"); 
+                }
+
+                referenceIndex++;
+                if (referenceIndex < model.MomentReferences.Length)
+                {
+                    // Proceed to next reference file.
+                    currentRef = CreateOpenReference();
+                    momentsRead = 0;
+                    return Next();
+                }
+                else
+                {
+                    // That was the last one.
+                    return null;
+                }
+            }
+            else
+            {
+                momentsRead++;
+                return moment;
+            }
+        }
+
+        public void Close()
+        {
+            if (currentRef != null)
+            {
+                currentRef.Close();
+                currentRef = null!;
+            }
+        }
+
+        private OpenReference CreateOpenReference()
+        {
+            var filepath = Path.Combine(workingDir, model.MomentReferences[referenceIndex].MomentsFile);
+            return new OpenReference(filepath);
+        }
+
+        private class OpenReference
+        {
+            private readonly FileStream file;
+            private readonly StreamReader reader;
+
+            public OpenReference(string filePath)
+            {
+                file = File.OpenRead(filePath);
+                reader = new StreamReader(file);
+            }
+
+            public OverwatchMoment? ReadNext()
+            {
+                var line = reader.ReadLine();
+                if (string.IsNullOrEmpty(line)) return null;
+                return JsonConvert.DeserializeObject<OverwatchMoment>(line);
+            }
+
+            public void Close()
+            {
+                reader.Close();
+                file.Close();
+
+                reader.Dispose();
+                file.Dispose();
+            }
+        }
+    }
+}

Framework/OverwatchTranscript/MomentReferenceBuilder.cs

@@ -0,0 +1,153 @@
+using Logging;
+using Newtonsoft.Json;
+
+namespace OverwatchTranscript
+{
+    public class MomentReferenceBuilder
+    {
+        private const int MaxMomentsPerReference = 10000;
+        private readonly ILog log;
+        private readonly string workingDir;
+
+        public MomentReferenceBuilder(ILog log, string workingDir)
+        {
+            this.log = log;
+            this.workingDir = workingDir;
+        }
+
+        public OverwatchMomentReference[] Build(IFinalizedBucket[] finalizedBuckets)
+        {
+            var result = new List<OverwatchMomentReference>();
+            var currentBuilder = new Builder(log, workingDir);
+
+            var buckets = finalizedBuckets.ToList();
+            log.Debug($"Building references for {buckets.Count} buckets.");
+            while (buckets.Any())
+            {
+                foreach (var b in buckets) b.Update();
+
+                buckets.RemoveAll(b => b.IsEmpty);
+                if (!buckets.Any()) break;
+
+                var earliestUtc = GetEarliestUtc(buckets);
+                if (earliestUtc == null) continue;
+                
+                var tops = CollectAllTopsForUtc(earliestUtc.Value, buckets);
+                var moment = ConvertTopsToMoment(tops);
+                currentBuilder.Add(moment);
+                if (currentBuilder.NumberOfMoments == MaxMomentsPerReference)
+                {
+                    result.Add(currentBuilder.Build());
+                    currentBuilder = new Builder(log, workingDir);
+                }
+            }
+
+            if (currentBuilder.NumberOfMoments > 0)
+            {
+                result.Add(currentBuilder.Build());
+            }
+
+            return result.ToArray();
+        }
+
+        private OverwatchMoment ConvertTopsToMoment(List<BucketTop> tops)
+        {
+            var discintUtc = tops.Select(e => e.Utc).Distinct().ToArray();
+            if (discintUtc.Length != 1) throw new Exception("UTC mixing in moment construction.");
+
+            return new OverwatchMoment
+            {
+                Utc = tops[0].Utc,
+                Events = tops.SelectMany(e => e.Events).ToArray()
+            };
+        }
+
+        private List<BucketTop> CollectAllTopsForUtc(DateTime earliestUtc, List<IFinalizedBucket> buckets)
+        {
+            var result = new List<BucketTop>();
+
+            foreach (var bucket in buckets)
+            {
+                if (bucket.IsEmpty) continue;
+
+                var utc = bucket.SeeTopUtc();
+                if (utc == null) continue;
+
+                if (utc.Value == earliestUtc)
+                {
+                    var top = bucket.TakeTop();
+                    if (top == null) throw new Exception("top was null after top utc was not");
+                    result.Add(top);
+                }
+            }
+
+            return result;
+        }
+
+        private DateTime? GetEarliestUtc(List<IFinalizedBucket> buckets)
+        {
+            var earliest = DateTime.MaxValue;
+            foreach (var bucket in buckets)
+            {
+                var utc = bucket.SeeTopUtc();
+                if (utc == null) return null;
+
+                if (utc.Value < earliest) earliest = utc.Value;
+            }
+            return earliest;
+        }
+
+        public class Builder
+        {
+            private readonly ILog log;
+            private readonly string workingDir;
+            private OverwatchMomentReference reference;
+            private readonly ActionQueue queue = new ActionQueue();
+
+            public Builder(ILog log, string workingDir)
+            {
+                reference = new OverwatchMomentReference
+                {
+                    MomentsFile = Guid.NewGuid().ToString(),
+                    EarliestUtc = DateTime.MaxValue,
+                    LatestUtc = DateTime.MinValue,
+                    NumberOfEvents = 0,
+                    NumberOfMoments = 0,
+                };
+                this.log = log;
+                this.workingDir = workingDir;
+                queue.Start();
+            }
+
+            public int NumberOfMoments => reference.NumberOfMoments;
+
+            public void Add(OverwatchMoment moment)
+            {
+                if (moment.Utc < reference.EarliestUtc) reference.EarliestUtc = moment.Utc;
+                if (moment.Utc > reference.LatestUtc) reference.LatestUtc = moment.Utc;
+                reference.NumberOfMoments++;
+                reference.NumberOfEvents += moment.Events.Length;
+
+                var filePath = Path.Combine(workingDir, reference.MomentsFile);
+
+                queue.Add(() =>
+                {
+                    File.AppendAllLines(filePath, new[]
+                    {
+                        Json.Serialize(moment)
+                    });
+                });
+            }
+
+            public OverwatchMomentReference Build()
+            {
+                queue.StopAndJoin();
+
+                log.Debug($"Created reference with {reference.NumberOfMoments} moments and {reference.NumberOfEvents} events...");
+                var result = reference;
+                reference = null!;
+                return result;
+            }
+        }
+    }
+}

Framework/OverwatchTranscript/OverwatchTranscript.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/OverwatchTranscript/Transcript.cs

@@ -0,0 +1,23 @@
+using Logging;
+
+namespace OverwatchTranscript
+{
+    public static class Transcript
+    {
+        public static ITranscriptWriter NewWriter(ILog log)
+        {
+            log = new LogPrefixer(log, "(TranscriptWriter) ");
+            return new TranscriptWriter(log, NewWorkDir());
+        }
+
+        public static ITranscriptReader NewReader(string transcriptFile)
+        {
+            return new TranscriptReader(NewWorkDir(), transcriptFile);
+        }
+
+        private static string NewWorkDir()
+        {
+            return Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+        }
+    }
+}

Framework/OverwatchTranscript/TranscriptConstants.cs

@@ -0,0 +1,8 @@
+namespace OverwatchTranscript
+{
+    public static class TranscriptConstants
+    {
+        public const string TranscriptFilename = "transcript.json";
+        public const string ArtifactFolderName = "artifacts";
+    }
+}

Framework/OverwatchTranscript/TranscriptReader.cs

@@ -0,0 +1,264 @@
+using Newtonsoft.Json;
+using System.IO;
+using System;
+using System.IO.Compression;
+using System.Linq;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+
+namespace OverwatchTranscript
+{
+    public interface ITranscriptReader
+    {
+        OverwatchCommonHeader Header { get; }
+        T GetHeader<T>(string key);
+        void AddMomentHandler(Action<ActivateMoment> handler);
+        void AddEventHandler<T>(Action<ActivateEvent<T>> handler);
+        bool Next();
+        void Close();
+    }
+
+    public class TranscriptReader : ITranscriptReader
+    {
+        private readonly object handlersLock = new object();
+        private readonly string transcriptFile;
+        private readonly string artifactsFolder;
+        private readonly List<Action<ActivateMoment>> momentHandlers = new List<Action<ActivateMoment>>();
+        private readonly Dictionary<string, List<Action<ActivateMoment, string>>> eventHandlers = new Dictionary<string, List<Action<ActivateMoment, string>>>();
+        private readonly string workingDir;
+        private readonly OverwatchTranscript model;
+        private bool closed;
+        private long momentCounter;
+        private readonly ConcurrentQueue<OverwatchMoment> queue = new ConcurrentQueue<OverwatchMoment>();
+        private readonly Task queueFiller;
+
+        public TranscriptReader(string workingDir, string inputFilename)
+        {
+            closed = false;
+            this.workingDir = workingDir;
+            transcriptFile = Path.Combine(workingDir, TranscriptConstants.TranscriptFilename);
+            artifactsFolder = Path.Combine(workingDir, TranscriptConstants.ArtifactFolderName);
+
+            if (!Directory.Exists(workingDir)) Directory.CreateDirectory(workingDir);
+            if (File.Exists(transcriptFile) || Directory.Exists(artifactsFolder)) throw new Exception("workingdir not clean");
+
+            model = LoadModel(inputFilename);
+
+            queueFiller = Task.Run(() => FillQueue(model, workingDir));
+        }
+
+        public OverwatchCommonHeader Header
+        {
+            get
+            {
+                CheckClosed();
+                return model.Header.Common;
+            }
+        }
+
+        public T GetHeader<T>(string key)
+        {
+            CheckClosed();
+            var value = model.Header.Entries.First(e => e.Key == key).Value;
+            return JsonConvert.DeserializeObject<T>(value)!;
+        }
+
+        public void AddMomentHandler(Action<ActivateMoment> handler)
+        {
+            CheckClosed();
+            lock (handlersLock)
+            {
+                momentHandlers.Add(handler);
+            }
+        }
+
+        public void AddEventHandler<T>(Action<ActivateEvent<T>> handler)
+        {
+            CheckClosed();
+
+            var typeName = typeof(T).FullName;
+            if (string.IsNullOrEmpty(typeName)) throw new Exception("Empty typename for payload");
+
+            lock (handlersLock)
+            {
+                if (eventHandlers.ContainsKey(typeName))
+                {
+                    eventHandlers[typeName].Add(CreateEventAction(handler));
+                }
+                else
+                {
+                    eventHandlers.Add(typeName, new List<Action<ActivateMoment, string>>
+                    {
+                        CreateEventAction(handler)
+                    });
+                }
+            }
+        }
+
+        private readonly object nextLock = new object();
+        private OverwatchMoment? moment = null;
+        private OverwatchMoment? next = null;
+
+        public bool Next()
+        {
+            CheckClosed();
+
+            OverwatchMoment? m = null;
+            TimeSpan? duration = null;
+            lock (nextLock)
+            {
+                if (next == null)
+                {
+                    if (!queue.TryDequeue(out moment)) return false;
+                    queue.TryDequeue(out next);
+                }
+                else
+                {
+                    moment = next;
+                    next = null;
+                    queue.TryDequeue(out next);
+                }
+
+                m = moment;
+                duration = GetMomentDuration();
+            }
+
+            ActivateMoment(moment, duration);
+
+            return true;
+        }
+
+        public void Close()
+        {
+            CheckClosed();
+            closed = true;
+
+            queueFiller.Wait();
+
+            Directory.Delete(workingDir, true);
+        }
+
+        private Action<ActivateMoment, string> CreateEventAction<T>(Action<ActivateEvent<T>> handler)
+        {
+            return (m, s) =>
+            {
+                handler(new ActivateEvent<T>(m, JsonConvert.DeserializeObject<T>(s)!));
+            };
+        }
+
+        private void FillQueue(OverwatchTranscript model, string workingDir)
+        {
+            var reader = new MomentReader(model, workingDir);
+
+            while (true)
+            {
+                if (closed)
+                {
+                    reader.Close();
+                    return;
+                }
+
+                while (queue.Count < 10)
+                {
+                    var moment = reader.Next();
+                    if (moment == null)
+                    {
+                        reader.Close();
+                        return;
+                    }
+                    queue.Enqueue(moment);
+                }
+
+                Thread.Sleep(1);
+            }
+        }
+
+        private TimeSpan? GetMomentDuration()
+        {
+            if (moment == null) return null;
+            if (next == null) return null;
+
+            return next.Utc - moment.Utc;
+        }
+
+        private void ActivateMoment(OverwatchMoment moment, TimeSpan? duration)
+        {
+            var m = new ActivateMoment(moment.Utc, duration, momentCounter);
+
+            lock (handlersLock)
+            {
+                ActivateMomentHandlers(m);
+
+                foreach (var @event in moment.Events)
+                {
+                    ActivateEventHandlers(m, @event);
+                }
+            }
+
+            momentCounter++;
+        }
+
+        private void ActivateMomentHandlers(ActivateMoment m)
+        {
+            foreach (var handler in momentHandlers)
+            {
+                handler(m);
+            }
+        }
+
+        private void ActivateEventHandlers(ActivateMoment m, OverwatchEvent @event)
+        {
+            if (!eventHandlers.ContainsKey(@event.Type)) return;
+            var handlers = eventHandlers[@event.Type];
+
+            foreach (var handler in handlers)
+            {
+                handler(m, @event.Payload);
+            }
+        }
+
+        private OverwatchTranscript LoadModel(string inputFilename)
+        {
+            ZipFile.ExtractToDirectory(inputFilename, workingDir);
+
+            if (!File.Exists(transcriptFile))
+            {
+                closed = true;
+                throw new Exception("Is not a transcript file. Unzipped to: " + workingDir);
+            }
+
+            return JsonConvert.DeserializeObject<OverwatchTranscript>(File.ReadAllText(transcriptFile))!;
+        }
+
+        private void CheckClosed()
+        {
+            if (closed) throw new Exception("Transcript has already been closed.");
+        }
+    }
+
+    public class ActivateMoment
+    {
+        public ActivateMoment(DateTime utc, TimeSpan? duration, long index)
+        {
+            Utc = utc;
+            Duration = duration;
+            Index = index;
+        }
+
+        public DateTime Utc { get; }
+        public TimeSpan? Duration { get; }
+        public long Index { get; }
+    }
+
+    public class ActivateEvent<T>
+    {
+        public ActivateEvent(ActivateMoment moment, T payload)
+        {
+            Moment = moment;
+            Payload = payload;
+        }
+
+        public ActivateMoment Moment { get; }
+        public T Payload { get; }
+    }
+}

Framework/OverwatchTranscript/TranscriptWriter.cs

@@ -0,0 +1,130 @@
+using Logging;
+using Newtonsoft.Json;
+using System.IO.Compression;
+
+namespace OverwatchTranscript
+{
+    public interface ITranscriptWriter
+    {
+        void AddHeader(string key, object value);
+        void Add(DateTime utc, object payload);
+        void IncludeArtifact(string filePath);
+        void Write(string outputFilename);
+    }
+
+    public class TranscriptWriter : ITranscriptWriter
+    {
+        private readonly object _lock = new object();
+        private readonly MomentReferenceBuilder builder;
+        private readonly string transcriptFile;
+        private readonly string artifactsFolder;
+        private readonly Dictionary<string, string> header = new Dictionary<string, string>();
+        private readonly BucketSet bucketSet;
+        private readonly ILog log;
+        private readonly string workingDir;
+        private bool closed;
+
+        public TranscriptWriter(ILog log, string workingDir)
+        {
+            closed = false;
+            this.log = log;
+            this.workingDir = workingDir;
+            bucketSet = new BucketSet(log, workingDir);
+            builder = new MomentReferenceBuilder(log, workingDir);
+            transcriptFile = Path.Combine(workingDir, TranscriptConstants.TranscriptFilename);
+            artifactsFolder = Path.Combine(workingDir, TranscriptConstants.ArtifactFolderName);
+
+            if (!Directory.Exists(workingDir)) Directory.CreateDirectory(workingDir);
+            if (File.Exists(transcriptFile) || Directory.Exists(artifactsFolder)) throw new Exception("workingdir not clean");
+        }
+
+        public void Add(DateTime utc, object payload)
+        {
+            CheckClosed();
+            bucketSet.Add(utc, payload);
+        }
+
+        public void AddHeader(string key, object value)
+        {
+            CheckClosed();
+            lock (_lock)
+            {
+                header.Add(key, Json.Serialize(value));
+            }
+        }
+
+        public void IncludeArtifact(string filePath)
+        {
+            CheckClosed();
+            if (!File.Exists(filePath)) throw new Exception("File not found: " + filePath);
+            if (!Directory.Exists(artifactsFolder)) Directory.CreateDirectory(artifactsFolder);
+            var name = Path.GetFileName(filePath);
+            File.Copy(filePath, Path.Combine(artifactsFolder, name), overwrite: false);
+        }
+
+        public void Write(string outputFilename)
+        {
+            CheckClosed();
+            closed = true;
+
+            var momentReferences = builder.Build(bucketSet.FinalizeBuckets());
+            var model = CreateModel(momentReferences);
+
+            File.WriteAllText(transcriptFile, Json.Serialize(model, Formatting.Indented));
+
+            ZipFile.CreateFromDirectory(workingDir, outputFilename);
+            log.Debug($"Transcript written to {outputFilename}");
+            log.Debug($"Common header: {Json.Serialize(model.Header.Common, Formatting.Indented)}");
+
+            Directory.Delete(workingDir, true);
+            log.Debug($"Workdir {workingDir} deleted");
+        }
+
+        private OverwatchTranscript CreateModel(OverwatchMomentReference[] momentReferences)
+        {
+            lock (_lock)
+            {
+                var model = new OverwatchTranscript
+                {
+                    Header = new OverwatchHeader
+                    {
+                        Common = CreateCommonHeader(momentReferences),
+                        Entries = header.Select(h =>
+                        {
+                            return new OverwatchHeaderEntry
+                            {
+                                Key = h.Key,
+                                Value = h.Value
+                            };
+                        }).ToArray()
+                    },
+                    MomentReferences = momentReferences
+                };
+
+                header.Clear();
+                return model;
+            }
+        }
+
+        private OverwatchCommonHeader CreateCommonHeader(OverwatchMomentReference[] momentReferences)
+        {
+            var moments = momentReferences.Sum(m => m.NumberOfMoments);
+            var events = momentReferences.Sum(m => m.NumberOfEvents);
+            var earliest = momentReferences.Min(m => m.EarliestUtc);
+            var latest = momentReferences.Max(m => m.LatestUtc);
+
+            return new OverwatchCommonHeader
+            {
+                NumberOfMoments = moments,
+                NumberOfEvents = events,
+                EarliestUtc = earliest,
+                LatestUtc = latest
+            };
+        }
+
+        private void CheckClosed()
+        {
+            if (closed) throw new Exception("Transcript has already been written. Cannot modify or write again.");
+        }
+    }
+}

Framework/Utils/BlockInterval.cs

@@ -15,12 +15,13 @@
                 To = from;
             }
             TimeRange = timeRange;
+            NumberOfBlocks = (To - From) + 1;
         }
 
         public ulong From { get; }
         public ulong To { get; }
         public TimeRange TimeRange { get; }
-        public ulong NumberOfBlocks => To - From;
+        public ulong NumberOfBlocks { get; }
 
         public override string ToString()
         {

Framework/Utils/ByteSize.cs

@@ -13,7 +13,6 @@
 
         public long SizeInBytes { get; }
 
-
         public long ToMB()
         {
             return SizeInBytes / (1024 * 1024);

Framework/Utils/PluginPathUtils.cs

@@ -0,0 +1,41 @@
+namespace Utils
+{
+    public static class PluginPathUtils
+    {
+        private const string ProjectPluginsFolderName = "ProjectPlugins";
+        private static string projectPluginsDir = string.Empty;
+
+        public static string ProjectPluginsDir
+        {
+            get
+            {
+                if (string.IsNullOrEmpty(projectPluginsDir)) projectPluginsDir = FindProjectPluginsDir();
+                return projectPluginsDir;
+            }
+        }
+
+        private static string FindProjectPluginsDir()
+        {
+            var current = Directory.GetCurrentDirectory();
+            while (true)
+            {
+                var localFolders = Directory.GetDirectories(current);
+                var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
+                if (projectPluginsFolders.Length == 1)
+                {
+                    return projectPluginsFolders.Single();
+                }
+
+                var parent = Directory.GetParent(current);
+                if (parent == null)
+                {
+                    var msg = $"Unable to locate '{ProjectPluginsFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
+                    Console.WriteLine(msg);
+                    throw new Exception(msg);
+                }
+
+                current = parent.FullName;
+            }
+        }
+    }
+}

Framework/Utils/RandomUtils.cs

@@ -3,13 +3,41 @@
     public static class RandomUtils
     {
         private static readonly Random random = new Random();
+        private static readonly object @lock = new object();
+
+        public static T GetOneRandom<T>(this T[] items)
+        {
+            lock (@lock)
+            {
+                var i = random.Next(0, items.Length);
+                var result = items[i];
+                return result;
+            }
+        }
 
         public static T PickOneRandom<T>(this List<T> remainingItems)
         {
-            var i = random.Next(0, remainingItems.Count);
-            var result = remainingItems[i];
-            remainingItems.RemoveAt(i);
-            return result;
+            lock (@lock)
+            {
+                var i = random.Next(0, remainingItems.Count);
+                var result = remainingItems[i];
+                remainingItems.RemoveAt(i);
+                return result;
+            }
+        }
+
+        public static T[] Shuffled<T>(T[] items)
+        {
+            lock (@lock)
+            {
+                var result = new List<T>();
+                var source = items.ToList();
+                while (source.Any())
+                {
+                    result.Add(RandomUtils.PickOneRandom(source));
+                }
+                return result.ToArray();
+            }
         }
     }
 }

Framework/Utils/Retry.cs

@@ -71,6 +71,10 @@
                         task();
                         return;
                     }
+                    catch (OperationCanceledException)
+                    {
+                        return;
+                    }
                     catch (Exception ex)
                     {
                         var failure = CaptureFailure(ex);
@@ -94,7 +98,7 @@
 
             private void Fail()
             {
-                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport}",
+                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport()}",
                         new AggregateException(failures.Select(f => f.Exception)));
             }
 

Framework/Utils/RollingAverage.cs

@@ -0,0 +1,28 @@
+namespace Utils
+{
+    public static class RollingAverage
+    {
+        /// <param name="currentAverage">Value of average before new value is added.</param>
+        /// <param name="newNumberOfValues">Number of values in average after new value is added.</param>
+        /// <param name="newValue">New value to be added.</param>
+        /// <returns>New average value.</returns>
+        /// <exception cref="Exception">newNumberOfValues must be 1 or greater.</exception>
+        public static float GetNewAverage(float currentAverage, int newNumberOfValues, float newValue)
+        {
+            if (newNumberOfValues < 1) throw new Exception("Should be at least 1 value.");
+
+            float n = newNumberOfValues;
+            var originalValue = currentAverage;
+            var originalValueWeight = ((n - 1.0f) / n);
+            var newValueWeight = (1.0f / n);
+            return GetWeightedAverage(originalValue, originalValueWeight, newValue, newValueWeight);
+        }
+
+        public static float GetWeightedAverage(float value1, float weight1, float value2, float weight2)
+        {
+            float totalWeight = weight1 + weight2;
+            if (totalWeight == 0.0f) return 0.0f;
+            return ((value1 * weight1) + (value2 * weight2)) / totalWeight;
+        }
+    }
+}

Framework/Utils/Str.cs

@@ -0,0 +1,13 @@
+namespace Utils
+{
+    public static class Str
+    {
+        public static string Between(string input, string open, string close)
+        {
+            var openIndex = input.IndexOf(open) + open.Length;
+            var closeIndex = input.LastIndexOf(close);
+
+            return input.Substring(openIndex, closeIndex - openIndex);
+        }
+    }
+}

Framework/Utils/Time.cs

@@ -33,6 +33,11 @@
             result += $"{d.Seconds} secs";
             return result;
         }
+
+        public static string FormatTimestamp(DateTime d)
+        {
+            return d.ToString("o");
+        }
         
         public static TimeSpan ParseTimespan(string span)
         {

Framework/Utils/Utils.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Utils</RootNamespace>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin.Marketplace;
+using System.Collections.Generic;
 using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
@@ -10,38 +11,33 @@ namespace CodexContractsPlugin.ChainMonitor
             Request[] requests,
             RequestFulfilledEventDTO[] fulfilled,
             RequestCancelledEventDTO[] cancelled,
+            RequestFailedEventDTO[] failed,
             SlotFilledEventDTO[] slotFilled,
-            SlotFreedEventDTO[] slotFreed
+            SlotFreedEventDTO[] slotFreed,
+            SlotReservationsFullEventDTO[] slotReservationsFull
             )
         {
             BlockInterval = blockInterval;
             Requests = requests;
             Fulfilled = fulfilled;
             Cancelled = cancelled;
+            Failed = failed;
             SlotFilled = slotFilled;
             SlotFreed = slotFreed;
+            SlotReservationsFull = slotReservationsFull;
+
+            All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull);
         }
 
         public BlockInterval BlockInterval { get; }
         public Request[] Requests { get; }
         public RequestFulfilledEventDTO[] Fulfilled { get; }
         public RequestCancelledEventDTO[] Cancelled { get; }
+        public RequestFailedEventDTO[] Failed { get; }
         public SlotFilledEventDTO[] SlotFilled { get; }
         public SlotFreedEventDTO[] SlotFreed { get; }
-
-        public IHasBlock[] All
-        {
-            get
-            {
-                var all = new List<IHasBlock>();
-                all.AddRange(Requests);
-                all.AddRange(Fulfilled);
-                all.AddRange(Cancelled);
-                all.AddRange(SlotFilled);
-                all.AddRange(SlotFreed);
-                return all.ToArray();
-            }
-        }
+        public SlotReservationsFullEventDTO[] SlotReservationsFull { get; }
+        public IHasBlock[] All { get; }
 
         public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
         {
@@ -60,9 +56,21 @@ namespace CodexContractsPlugin.ChainMonitor
                 events.GetStorageRequests(),
                 events.GetRequestFulfilledEvents(),
                 events.GetRequestCancelledEvents(),
+                events.GetRequestFailedEvents(),
                 events.GetSlotFilledEvents(),
-                events.GetSlotFreedEvents()
+                events.GetSlotFreedEvents(),
+                events.GetSlotReservationsFull()
             );
         }
+
+        private T[] ConcatAll<T>(params T[][] arrays)
+        {
+            var result = Array.Empty<T>();
+            foreach (var array in arrays)
+            {
+                result = result.Concat(array).ToArray();
+            }
+            return result;
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -1,4 +1,6 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
 using Logging;
 using System.Numerics;
 using Utils;
@@ -7,12 +9,28 @@ namespace CodexContractsPlugin.ChainMonitor
 {
     public interface IChainStateChangeHandler
     {
-        void OnNewRequest(IChainStateRequest request);
-        void OnRequestFinished(IChainStateRequest request);
-        void OnRequestFulfilled(IChainStateRequest request);
-        void OnRequestCancelled(IChainStateRequest request);
-        void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex);
-        void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex);
+        void OnNewRequest(RequestEvent requestEvent);
+        void OnRequestFinished(RequestEvent requestEvent);
+        void OnRequestFulfilled(RequestEvent requestEvent);
+        void OnRequestCancelled(RequestEvent requestEvent);
+        void OnRequestFailed(RequestEvent requestEvent);
+        void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
+        void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
+        void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
+
+        void OnError(string msg);
+    }
+
+    public class RequestEvent
+    {
+        public RequestEvent(BlockTimeEntry block, IChainStateRequest request)
+        {
+            Block = block;
+            Request = request;
+        }
+
+        public BlockTimeEntry Block { get; }
+        public IChainStateRequest Request { get; }
     }
 
     public class ChainState
@@ -27,39 +45,42 @@ namespace CodexContractsPlugin.ChainMonitor
             this.log = new LogPrefixer(log, "(ChainState) ");
             this.contracts = contracts;
             handler = changeHandler;
-            StartUtc = startUtc;
             TotalSpan = new TimeRange(startUtc, startUtc);
         }
 
         public TimeRange TotalSpan { get; private set; }
         public IChainStateRequest[] Requests => requests.ToArray();
 
-        public DateTime StartUtc { get; }
-
-        public void Update()
+        public int Update()
         {
-            Update(DateTime.UtcNow);
+            return Update(DateTime.UtcNow);
         }
 
-        public void Update(DateTime toUtc)
+        public int Update(DateTime toUtc)
         {
             var span = new TimeRange(TotalSpan.To, toUtc);
             var events = ChainEvents.FromTimeRange(contracts, span);
             Apply(events);
 
             TotalSpan = new TimeRange(TotalSpan.From, span.To);
+            return events.All.Length;
         }
 
         private void Apply(ChainEvents events)
         {
             if (events.BlockInterval.TimeRange.From < TotalSpan.From)
-                throw new Exception("Attempt to update ChainState with set of events from before its current record.");
+            {
+                var msg = "Attempt to update ChainState with set of events from before its current record.";
+                handler.OnError(msg);
+                throw new Exception(msg);
+            }
 
-            log.Log($"ChainState updating: {events.BlockInterval}");
+            log.Log($"ChainState updating: {events.BlockInterval} = {events.All.Length} events.");
 
             // Run through each block and apply the events to the state in order.
             var span = events.BlockInterval.TimeRange.Duration;
             var numBlocks = events.BlockInterval.NumberOfBlocks;
+            if (numBlocks == 0) return;
             var spanPerBlock = span / numBlocks;
 
             var eventUtc = events.BlockInterval.TimeRange.From;
@@ -91,41 +112,57 @@ namespace CodexContractsPlugin.ChainMonitor
             var newRequest = new ChainStateRequest(log, request, RequestState.New);
             requests.Add(newRequest);
 
-            handler.OnNewRequest(newRequest);
+            handler.OnNewRequest(new RequestEvent(request.Block, newRequest));
         }
 
-        private void ApplyEvent(RequestFulfilledEventDTO request)
+        private void ApplyEvent(RequestFulfilledEventDTO @event)
         {
-            var r = FindRequest(request.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
-            r.UpdateState(request.Block.BlockNumber, RequestState.Started);
-            handler.OnRequestFulfilled(r);
+            r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
+            handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
         }
 
-        private void ApplyEvent(RequestCancelledEventDTO request)
+        private void ApplyEvent(RequestCancelledEventDTO @event)
         {
-            var r = FindRequest(request.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
-            r.UpdateState(request.Block.BlockNumber, RequestState.Cancelled);
-            handler.OnRequestCancelled(r);
+            r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
+            handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
         }
 
-        private void ApplyEvent(SlotFilledEventDTO request)
+        private void ApplyEvent(RequestFailedEventDTO @event)
         {
-            var r = FindRequest(request.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
-            r.Hosts.Add(request.Host, (int)request.SlotIndex);
-            r.Log($"[{request.Block.BlockNumber}] SlotFilled (host:'{request.Host}', slotIndex:{request.SlotIndex})");
-            handler.OnSlotFilled(r, request.SlotIndex);
+            r.UpdateState(@event.Block.BlockNumber, RequestState.Failed);
+            handler.OnRequestFailed(new RequestEvent(@event.Block, r));
         }
 
-        private void ApplyEvent(SlotFreedEventDTO request)
+        private void ApplyEvent(SlotFilledEventDTO @event)
         {
-            var r = FindRequest(request.RequestId);
+            var r = FindRequest(@event);
             if (r == null) return;
-            r.Hosts.RemoveHost((int)request.SlotIndex);
-            r.Log($"[{request.Block.BlockNumber}] SlotFreed (slotIndex:{request.SlotIndex})");
-            handler.OnSlotFreed(r, request.SlotIndex);
+            r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
+            r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
+            handler.OnSlotFilled(new RequestEvent(@event.Block, r), @event.Host, @event.SlotIndex);
+        }
+
+        private void ApplyEvent(SlotFreedEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.Hosts.RemoveHost((int)@event.SlotIndex);
+            r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
+            handler.OnSlotFreed(new RequestEvent(@event.Block, r), @event.SlotIndex);
+        }
+
+        private void ApplyEvent(SlotReservationsFullEventDTO @event)
+        {
+            var r = FindRequest(@event);
+            if (r == null) return;
+            r.Log($"[{@event.Block.BlockNumber}] SlotReservationsFull (slotIndex:{@event.SlotIndex})");
+            handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
         }
 
         private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
@@ -136,15 +173,28 @@ namespace CodexContractsPlugin.ChainMonitor
                     && r.FinishedUtc < eventsUtc)
                 {
                     r.UpdateState(blockNumber, RequestState.Finished);
-                    handler.OnRequestFinished(r);
+                    handler.OnRequestFinished(new RequestEvent(new BlockTimeEntry(blockNumber, eventsUtc), r));
                 }
             }
         }
 
-        private ChainStateRequest? FindRequest(byte[] requestId)
+        private ChainStateRequest? FindRequest(IHasRequestId request)
         {
-            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
-            if (r == null) log.Log("Unable to find request by ID!");
+            var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
+            if (r == null)
+            {
+                var blockNumber = "unknown";
+                if (request is IHasBlock blk)
+                {
+                    blockNumber = blk.Block.BlockNumber.ToString();
+                }
+
+                var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
+                    $"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
+
+                log.Error(msg);
+                handler.OnError(msg);
+            }
             return r;
         }
 

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -0,0 +1,60 @@
+using GethPlugin;
+using System.Numerics;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public class ChainStateChangeHandlerMux : IChainStateChangeHandler
+    {
+        public ChainStateChangeHandlerMux(params IChainStateChangeHandler[] handlers)
+        {
+            Handlers = handlers.ToList();
+        }
+
+        public List<IChainStateChangeHandler> Handlers { get; } = new List<IChainStateChangeHandler>();
+
+        public void OnNewRequest(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnNewRequest(requestEvent);
+        }
+
+        public void OnRequestCancelled(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnRequestCancelled(requestEvent);
+        }
+
+        public void OnRequestFailed(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnRequestFailed(requestEvent);
+        }
+
+        public void OnRequestFinished(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnRequestFinished(requestEvent);
+        }
+
+        public void OnRequestFulfilled(RequestEvent requestEvent)
+        {
+            foreach (var handler in Handlers) handler.OnRequestFulfilled(requestEvent);
+        }
+
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotFilled(requestEvent, host, slotIndex);
+        }
+
+        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotFreed(requestEvent, slotIndex);
+        }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+            foreach (var handler in Handlers) handler.OnSlotReservationsFull(requestEvent, slotIndex);
+        }
+
+        public void OnError(string msg)
+        {
+            foreach (var handler in Handlers) handler.OnError(msg);
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -1,30 +1,43 @@
-using System.Numerics;
+using GethPlugin;
+using System.Numerics;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
     public class DoNothingChainEventHandler : IChainStateChangeHandler
     {
-        public void OnNewRequest(IChainStateRequest request)
+        public void OnNewRequest(RequestEvent requestEvent)
         {
         }
 
-        public void OnRequestCancelled(IChainStateRequest request)
+        public void OnRequestCancelled(RequestEvent requestEvent)
         {
         }
 
-        public void OnRequestFinished(IChainStateRequest request)
+        public void OnRequestFailed(RequestEvent requestEvent)
         {
         }
 
-        public void OnRequestFulfilled(IChainStateRequest request)
+        public void OnRequestFinished(RequestEvent requestEvent)
         {
         }
 
-        public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex)
+        public void OnRequestFulfilled(RequestEvent requestEvent)
         {
         }
 
-        public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex)
+        public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex)
+        {
+        }
+
+        public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
+        {
+        }
+
+        public void OnError(string msg)
         {
         }
     }

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -1,7 +1,9 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.ABI;
+using Nethereum.Hex.HexConvertors.Extensions;
 using Nethereum.Util;
 using NethereumWorkflow;
 using Newtonsoft.Json;
@@ -24,6 +26,7 @@ namespace CodexContractsPlugin
         ICodexContractsEvents GetEvents(BlockInterval blockInterval);
         EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
         RequestState GetRequestState(Request request);
+        void WaitUntilNextPeriod();
     }
 
     [JsonConverter(typeof(StringEnumConverter))]
@@ -114,6 +117,15 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
+        public void WaitUntilNextPeriod()
+        {
+            log.Log("Waiting until next proof period...");
+            var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
+            var periodSeconds = (int)Deployment.Config.Proofs.Period;
+            var secondsLeft = now % periodSeconds;
+            Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1));
+        }
+
         private ContractInteractions StartInteraction()
         {
             return new ContractInteractions(log, gethNode);

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -19,7 +19,7 @@ namespace CodexContractsPlugin
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 
-            var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
+            var address = config.GethNode.StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
 
             SetSchedulingAffinity(notIn: "false");
 

ProjectPlugins/CodexContractsPlugin/CodexContractsDeployment.cs

@@ -1,14 +1,18 @@
-namespace CodexContractsPlugin
+using CodexContractsPlugin.Marketplace;
+
+namespace CodexContractsPlugin
 {
     public class CodexContractsDeployment
     {
-        public CodexContractsDeployment(string marketplaceAddress, string abi, string tokenAddress)
+        public CodexContractsDeployment(MarketplaceConfig config, string marketplaceAddress, string abi, string tokenAddress)
         {
+            Config = config;
             MarketplaceAddress = marketplaceAddress;
             Abi = abi;
             TokenAddress = tokenAddress;
         }
 
+        public MarketplaceConfig Config { get; }
         public string MarketplaceAddress { get; }
         public string Abi { get; }
         public string TokenAddress { get; }

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -1,8 +1,9 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
+using Nethereum.Contracts;
 using Nethereum.Hex.HexTypes;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -13,8 +14,10 @@ namespace CodexContractsPlugin
         Request[] GetStorageRequests();
         RequestFulfilledEventDTO[] GetRequestFulfilledEvents();
         RequestCancelledEventDTO[] GetRequestCancelledEvents();
+        RequestFailedEventDTO[] GetRequestFailedEvents();
         SlotFilledEventDTO[] GetSlotFilledEvents();
         SlotFreedEventDTO[] GetSlotFreedEvents();
+        SlotReservationsFullEventDTO[] GetSlotReservationsFull();
     }
 
     public class CodexContractsEvents : ICodexContractsEvents
@@ -37,38 +40,32 @@ namespace CodexContractsPlugin
         {
             var events = gethNode.GetEvents<StorageRequestedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
             var i = new ContractInteractions(log, gethNode);
-            return events
-            .Select(e =>
+            return events.Select(e =>
             {
-                        var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
-                        var result = requestEvent.ReturnValue1;
-                        result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                        result.RequestId = e.Event.RequestId;
-                        return result;
-                    })
-                    .ToArray();
+                var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId);
+                var result = requestEvent.ReturnValue1;
+                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+                result.RequestId = e.Event.RequestId;
+                return result;
+            }).ToArray();
         }
 
         public RequestFulfilledEventDTO[] GetRequestFulfilledEvents()
         {
             var events = gethNode.GetEvents<RequestFulfilledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public RequestCancelledEventDTO[] GetRequestCancelledEvents()
         {
             var events = gethNode.GetEvents<RequestCancelledEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        public RequestFailedEventDTO[] GetRequestFailedEvents()
+        {
+            var events = gethNode.GetEvents<RequestFailedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
         }
 
         public SlotFilledEventDTO[] GetSlotFilledEvents()
@@ -86,12 +83,20 @@ namespace CodexContractsPlugin
         public SlotFreedEventDTO[] GetSlotFreedEvents()
         {
             var events = gethNode.GetEvents<SlotFreedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
-            return events.Select(e =>
-            {
-                var result = e.Event;
-                result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
-                return result;
-            }).ToArray();
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        public SlotReservationsFullEventDTO[] GetSlotReservationsFull()
+        {
+            var events = gethNode.GetEvents<SlotReservationsFullEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
+        private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
+        {
+            var result = e.Event;
+            result.Block = GetBlock(e.Log.BlockNumber.ToUlong());
+            return result;
         }
 
         private BlockTimeEntry GetBlock(ulong number)

ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj

@@ -1,11 +1,16 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="Nethereum.Generators" Version="4.21.4" />
+    <PackageReference Include="Nethereum.Generators.Net" Version="4.21.4" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
     <ProjectReference Include="..\GethPlugin\GethPlugin.csproj" />

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -1,8 +1,10 @@
-using Core;
+using CodexContractsPlugin.Marketplace;
+using Core;
 using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
+using Newtonsoft.Json;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -33,6 +35,7 @@ namespace CodexContractsPlugin
             try
             {
                 var result = DeployContract(container, workflow, gethNode);
+
                 workflow.Stop(containers, waitTillStopped: false);
                 Log("Container stopped.");
                 return result;
@@ -64,7 +67,8 @@ namespace CodexContractsPlugin
 
             var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container);
             var marketplaceAddress = extractor.ExtractMarketplaceAddress();
-            var abi = extractor.ExtractMarketplaceAbi();
+            var (abi, bytecode) = extractor.ExtractMarketplaceAbiAndByteCode();
+            EnsureCompatbility(abi, bytecode);
 
             var interaction = new ContractInteractions(tools.GetLog(), gethNode);
             var tokenAddress = interaction.GetTokenAddress(marketplaceAddress);
@@ -73,9 +77,32 @@ namespace CodexContractsPlugin
 
             Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract));
 
-            Log("Synced. Codex SmartContracts deployed.");
+            Log("Synced. Codex SmartContracts deployed. Getting configuration...");
 
-            return new CodexContractsDeployment(marketplaceAddress, abi, tokenAddress);
+            var config = GetMarketplaceConfiguration(marketplaceAddress, gethNode);
+
+            Log("Got config: " + JsonConvert.SerializeObject(config));
+
+            return new CodexContractsDeployment(config, marketplaceAddress, abi, tokenAddress);
+        }
+
+        private MarketplaceConfig GetMarketplaceConfiguration(string marketplaceAddress, IGethNode gethNode)
+        {
+            var func = new ConfigurationFunctionBase();
+            var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
+            return response.ReturnValue1;
+        }
+
+        private void EnsureCompatbility(string abi, string bytecode)
+        {
+            var expectedByteCode = MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant();
+
+            if (bytecode != expectedByteCode)
+            {
+                Log("Deployed contract is incompatible with current build of CodexContracts plugin. Running self-updater...");
+                var selfUpdater = new SelfUpdater();
+                selfUpdater.Update(abi, bytecode);
+            }
         }
 
         private void Log(string msg)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
@@ -23,9 +24,8 @@ namespace CodexContractsPlugin
         public string GetTokenAddress(string marketplaceAddress)
         {
             log.Debug(marketplaceAddress);
-            var function = new GetTokenFunction();
-
-            return gethNode.Call<GetTokenFunction, string>(marketplaceAddress, function);
+            var function = new TokenFunctionBase();
+            return gethNode.Call<TokenFunctionBase, string>(marketplaceAddress, function);
         }
 
         public string GetTokenName(string tokenAddress)
@@ -111,11 +111,6 @@ namespace CodexContractsPlugin
         }
     }
 
-    [Function("token", "address")]
-    public class GetTokenFunction : FunctionMessage
-    {
-    }
-
     [Function("name", "string")]
     public class GetTokenNameFunction : FunctionMessage
     {

ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs

@@ -31,14 +31,14 @@ namespace CodexContractsPlugin
             return marketplaceAddress;
         }
 
-        public string ExtractMarketplaceAbi()
+        public (string, string) ExtractMarketplaceAbiAndByteCode()
         {
             log.Debug();
-            var marketplaceAbi = Retry(FetchMarketplaceAbi);
-            if (string.IsNullOrEmpty(marketplaceAbi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
+            var (abi, bytecode) = Retry(FetchMarketplaceAbiAndByteCode);
+            if (string.IsNullOrEmpty(abi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
 
-            log.Debug("Got Marketplace ABI: " + marketplaceAbi);
-            return marketplaceAbi;
+            log.Debug("Got Marketplace ABI: " + abi);
+            return (abi, bytecode);
         }
 
         private string FetchMarketplaceAddress()
@@ -48,7 +48,7 @@ namespace CodexContractsPlugin
             return marketplace!.address;
         }
 
-        private string FetchMarketplaceAbi()
+        private (string, string) FetchMarketplaceAbiAndByteCode()
         {
             var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceArtifactFilename);
 
@@ -56,19 +56,12 @@ namespace CodexContractsPlugin
             var abi = artifact["abi"];
             var byteCode = artifact["bytecode"];
             var abiResult = abi!.ToString(Formatting.None);
-            var byteCodeResult = byteCode!.ToString(Formatting.None);
-
-            if (byteCodeResult
-                .ToLowerInvariant()
-                .Replace("\"", "") != MarketplaceDeploymentBase.BYTECODE.ToLowerInvariant())
-            {
-                throw new Exception("BYTECODE in CodexContractsPlugin does not match BYTECODE deployed by container. Update Marketplace.cs generated code?");
-            }
-
-            return abiResult;
+            var byteCodeResult = byteCode!.ToString(Formatting.None).ToLowerInvariant().Replace("\"", "");
+          
+            return (abiResult, byteCodeResult);
         }
 
-        private static string Retry(Func<string> fetch)
+        private static T Retry<T>(Func<T> fetch)
         {
             return Time.Retry(fetch, nameof(ContractsContainerInfoExtractor));
         }

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,6 +1,6 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
+using BlockchainUtils;
 using GethPlugin;
-using NethereumWorkflow.BlockUtils;
 using Newtonsoft.Json;
 
 namespace CodexContractsPlugin.Marketplace
@@ -10,7 +10,12 @@ namespace CodexContractsPlugin.Marketplace
         BlockTimeEntry Block { get; set; }
     }
 
-    public partial class Request : RequestBase, IHasBlock
+    public interface IHasRequestId
+    {
+        byte[] RequestId { get; set; }
+    }
+
+    public partial class Request : RequestBase, IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
@@ -28,26 +33,38 @@ namespace CodexContractsPlugin.Marketplace
         }
     }
 
-    public partial class RequestFulfilledEventDTO : IHasBlock
+    public partial class RequestFulfilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class RequestCancelledEventDTO : IHasBlock
+    public partial class RequestCancelledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
 
-    public partial class SlotFilledEventDTO : IHasBlock
+    public partial class RequestFailedEventDTO : IHasBlock, IHasRequestId
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
+
+    public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
         public EthAddress Host { get; set; }
     }
 
-    public partial class SlotFreedEventDTO : IHasBlock
+    public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
+
+    public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId
     {
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }

ProjectPlugins/CodexContractsPlugin/SelfUpdater.cs

@@ -0,0 +1,96 @@
+using Utils;
+
+namespace CodexContractsPlugin
+{
+    public class SelfUpdater
+    {
+        public void Update(string abi, string bytecode)
+        {
+            var filePath = GetMarketplaceFilePath();
+            var content = GenerateContent(abi, bytecode);
+            var contentLines = content.Split("\r\n");
+
+            var beginWith = new string[]
+            {
+                "using Nethereum.ABI.FunctionEncoding.Attributes;",
+                "using Nethereum.Contracts;",
+                "using System.Numerics;",
+                "",
+                "// Generated code, do not modify.",
+                "",
+                "#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.",
+                "namespace CodexContractsPlugin.Marketplace",
+                "{"
+            };
+
+            var endWith = new string[]
+            {
+                "}",
+                "#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable."
+            };
+
+            File.Delete(filePath);
+            File.WriteAllLines(filePath,
+                beginWith.Concat(
+                contentLines.Concat(
+                endWith))
+            );
+
+            throw new Exception("Oh no! CodexContracts were updated. Current build of CodexContractsPlugin is incompatible. " +
+                "But fear not! SelfUpdater.cs has automatically updated the plugin. Just rebuild and rerun and it should work. " +
+                "Just in case, manual update instructions are found here: 'CodexContractsPlugin/Marketplace/README.md'.");
+        }
+
+        private string GetMarketplaceFilePath()
+        {
+            var projectPluginDir = PluginPathUtils.ProjectPluginsDir;
+            var path = Path.Combine(projectPluginDir, "CodexContractsPlugin", "Marketplace", "Marketplace.cs");
+            if (!File.Exists(path)) throw new Exception("Marketplace file not found. Expected: " + path);
+            return path;
+        }
+
+        private string GenerateContent(string abi, string bytecode)
+        {
+            var deserializer = new Nethereum.Generators.Net.GeneratorModelABIDeserialiser();
+            var abiModel = deserializer.DeserialiseABI(abi);
+            var abiCtor = abiModel.Constructor;
+            var c = new Nethereum.Generators.CQS.ContractDeploymentCQSMessageGenerator(abiCtor, "namespace", bytecode, "Marketplace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+            var lines = "";
+            lines += c.GenerateClass();
+            lines += "\r\n";
+
+            foreach (var eventAbi in abiModel.Events)
+            {
+                var d = new Nethereum.Generators.DTOs.EventDTOGenerator(eventAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += d.GenerateClass();
+                lines += "\r\n";
+            }
+
+            foreach (var errorAbi in abiModel.Errors)
+            {
+                var e = new Nethereum.Generators.DTOs.ErrorDTOGenerator(errorAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += e.GenerateClass();
+                lines += "\r\n";
+            }
+
+            foreach (var funcAbi in abiModel.Functions)
+            {
+                var f = new Nethereum.Generators.DTOs.FunctionOutputDTOGenerator(funcAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                var ff = new Nethereum.Generators.CQS.FunctionCQSMessageGenerator(funcAbi, "namespace", "funcoutput", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += f.GenerateClass();
+                lines += "\r\n";
+                lines += ff.GenerateClass();
+                lines += "\r\n";
+            }
+
+            foreach (var structAbi in abiModel.Structs)
+            {
+                var g = new Nethereum.Generators.DTOs.StructTypeGenerator(structAbi, "namespace", Nethereum.Generators.Core.CodeGenLanguage.CSharp);
+                lines += g.GenerateClass();
+                lines += "\r\n";
+            }
+
+            return lines;
+        }
+    }
+}

ProjectPlugins/CodexContractsPlugin/TestTokenExtensions.cs

@@ -46,6 +46,16 @@ namespace CodexContractsPlugin
             return new TestToken(a.TstWei + b.TstWei);
         }
 
+        public static TestToken operator -(TestToken a, TestToken b)
+        {
+            return new TestToken(a.TstWei - b.TstWei);
+        }
+
+        public static TestToken operator *(TestToken a, int b)
+        {
+            return new TestToken(a.TstWei * b);
+        }
+
         public static bool operator <(TestToken a, TestToken b)
         {
             return a.TstWei < b.TstWei;

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin
     public class DiscordBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-bibliotech";
-        public override string Image => "codexstorage/codex-discordbot:sha-22cf82b";
+        public override string Image => "codexstorage/codex-discordbot:sha-8033da1";
 
         public static string RewardsPort = "bot_rewards_port";
 

ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs

@@ -7,8 +7,7 @@ namespace CodexDiscordBotPlugin
     public class RewarderBotContainerRecipe : ContainerRecipeFactory
     {
         public override string AppName => "discordbot-rewarder";
-        public override string Image => "thatbenbierens/codex-rewardbot:newstate";
-            //"codexstorage/codex-rewarderbot:sha-12dc7ef";
+        public override string Image => "codexstorage/codex-rewarderbot:sha-8033da1";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -3,13 +3,14 @@ using KubernetesWorkflow.Types;
 using Logging;
 using System.Security.Cryptography;
 using System.Text;
+using Utils;
 
 namespace CodexPlugin
 {
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "67-76-AB-FC-54-4F-EB-81-F5-E4-F8-27-DF-82-92-41-63-A5-EA-1B-17-14-0C-BE-20-9C-B3-DF-CE-E4-AA-38";
+        private const string OpenApiYamlHash = "34-B5-DA-26-40-76-B8-D8-8E-7D-9C-17-85-C6-B0-63-55-8D-C6-01-0B-96-BB-7C-BD-53-E5-32-07-ED-29-92";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 
@@ -21,8 +22,9 @@ namespace CodexPlugin
 
         private const string Failure =
             "Codex API compatibility check failed! " +
-            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. Please update the openapi.yaml in " +
-            "'ProjectPlugins/CodexPlugin' and rebuild this project. If you wish to disable API compatibility checking, please set " +
+            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. The openapi.yaml in " +
+            "'ProjectPlugins/CodexPlugin' has been overwritten with the container one. " +
+            "Please and rebuild this project. If you wish to disable API compatibility checking, please set " +
             $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/CodexPlugin/ApiChecker.cs'.";
 
         private static bool checkPassed = false;
@@ -71,10 +73,23 @@ namespace CodexPlugin
                 return;
             }
 
+            OverwriteOpenApiYaml(containerApi);
+
             log.Error(Failure);
             throw new Exception(Failure);
         }
 
+        private void OverwriteOpenApiYaml(string containerApi)
+        {
+            Log("API compatibility check failed. Updating CodexPlugin...");
+            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin", "openapi.yaml");
+            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexPlugin/openapi.yaml. Expected: " + openApiFilePath);
+
+            File.Delete(openApiFilePath);
+            File.WriteAllText(openApiFilePath, containerApi);
+            Log("CodexPlugin/openapi.yaml has been updated.");
+        }
+
         private string Hash(string file)
         {
             var fileBytes = Encoding.ASCII.GetBytes(file

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -32,6 +32,22 @@ namespace CodexPlugin
             return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
         }
 
+        public string GetSpr()
+        {
+            return CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                var json = endpoint.HttpGetString("spr");
+                var response = JsonConvert.DeserializeObject<SprResponse>(json);
+                return response!.Spr;
+            });
+        }
+
+        private class SprResponse
+        {
+            public string Spr { get; set; } = string.Empty;
+        }
+
         public DebugPeer GetDebugPeer(string peerId)
         {
             // Cannot use openAPI: debug/peer endpoint is not specified there.
@@ -63,44 +79,75 @@ namespace CodexPlugin
             });
         }
 
-        public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
+        public string UploadFile(UploadInput uploadInput, Action<Failure> onFailure)
         {
             return OnCodex(
-                api => api.UploadAsync(fileStream),
+                api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream),
                 CreateRetryConfig(nameof(UploadFile), onFailure));
         }
 
         public Stream DownloadFile(string contentId, Action<Failure> onFailure)
         {
             var fileResponse = OnCodex(
-                api => api.DownloadNetworkAsync(contentId),
+                api => api.DownloadNetworkStreamAsync(contentId),
                 CreateRetryConfig(nameof(DownloadFile), onFailure));
 
             if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
             return fileResponse.Stream;
         }
 
+        public LocalDataset DownloadStreamless(ContentId cid)
+        {
+            var response = OnCodex(api => api.DownloadNetworkAsync(cid.Id));
+            return mapper.Map(response);
+        }
+
+        public LocalDataset DownloadManifestOnly(ContentId cid)
+        {
+            var response = OnCodex(api => api.DownloadNetworkManifestAsync(cid.Id));
+            return mapper.Map(response);
+        }
+
         public LocalDatasetList LocalFiles()
         {
-            return mapper.Map(OnCodex(api => api.ListDataAsync()));
+            // API for listData mismatches.
+            //return mapper.Map(OnCodex(api => api.ListDataAsync()));
+
+            return mapper.Map(CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                return Time.Retry(() =>
+                {
+                    var str = endpoint.HttpGetString("data");
+                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
+                    return JsonConvert.DeserializeObject<LocalDatasetListJson>(str)!;
+                }, nameof(LocalFiles));
+            }));
+
         }
 
         public StorageAvailability SalesAvailability(StorageAvailability request)
         {
             var body = mapper.Map(request);
-            var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
+            var read = OnCodex(api => api.OfferStorageAsync(body));
             return mapper.Map(read);
         }
 
+        public StorageAvailability[] GetAvailabilities()
+        {
+            var collection = OnCodex(api => api.GetAvailabilitiesAsync());
+            return mapper.Map(collection);
+        }
+
         public string RequestStorage(StoragePurchaseRequest request)
         {
             var body = mapper.Map(request);
-            return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
+            return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
         public CodexSpace Space()
         {
-            var space = OnCodex<Space>(api => api.SpaceAsync());
+            var space = OnCodex(api => api.SpaceAsync());
             return mapper.Map(space);
         }
 
@@ -132,19 +179,6 @@ namespace CodexPlugin
             return workflow.GetPodInfo(Container);
         }
 
-        public void LogDiskSpace(string msg)
-        {
-            try
-            {
-                var diskInfo = tools.CreateWorkflow().ExecuteCommand(Container.Containers.Single(), "df", "--sync");
-                Log($"{msg} - Disk info: {diskInfo}");
-            }
-            catch (Exception e)
-            {
-                Log("Failed to get disk info: " + e);
-            }
-        }
-
         public void DeleteRepoFolder()
         {
             try
@@ -163,13 +197,13 @@ namespace CodexPlugin
 
         private T OnCodex<T>(Func<CodexApi, Task<T>> action)
         {
-            var result = tools.CreateHttp(CheckContainerCrashed).OnClient(client => CallCodex(client, action));
+            var result = tools.CreateHttp(GetHttpId(), CheckContainerCrashed).OnClient(client => CallCodex(client, action));
             return result;
         }
 
         private T OnCodex<T>(Func<CodexApi, Task<T>> action, Retry retry)
         {
-            var result = tools.CreateHttp(CheckContainerCrashed).OnClient(client => CallCodex(client, action), retry);
+            var result = tools.CreateHttp(GetHttpId(), CheckContainerCrashed).OnClient(client => CallCodex(client, action), retry);
             return result;
         }
 
@@ -196,13 +230,25 @@ namespace CodexPlugin
         private IEndpoint GetEndpoint()
         {
             return tools
-                .CreateHttp(CheckContainerCrashed)
+                .CreateHttp(GetHttpId(), CheckContainerCrashed)
                 .CreateEndpoint(GetAddress(), "/api/codex/v1/", Container.Name);
         }
 
+        public static Address? UploaderOverride { get; set; } = null;
+        public static Address? DownloaderOverride { get; set; } = null;
+
         private Address GetAddress()
         {
-            return Container.Containers.Single().GetAddress(log, CodexContainerRecipe.ApiPortTag);
+            if (GetName().ToLowerInvariant().Contains("upload") && UploaderOverride != null) return UploaderOverride;
+            if (GetName().ToLowerInvariant().Contains("download") && DownloaderOverride != null) return DownloaderOverride;
+
+
+            return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
+        }
+
+        private string GetHttpId()
+        {
+            return GetAddress().ToString();
         }
 
         private void CheckContainerCrashed(HttpClient client)
@@ -227,8 +273,6 @@ namespace CodexPlugin
                 $"(HTTP timeout = {Time.FormatDuration(timeSet.HttpCallTimeout())}) " +
                 $"Checking if node responds to debug/info...");
 
-            LogDiskSpace("After retry failure");
-
             try
             {
                 var debugInfo = GetDebugInfo();
@@ -265,4 +309,18 @@ namespace CodexPlugin
             log.Log($"{GetName()} {msg}");
         }
     }
+
+    public class UploadInput
+    {
+        public UploadInput(string contentType, string contentDisposition, FileStream fileStream)
+        {
+            ContentType = contentType;
+            ContentDisposition = contentDisposition;
+            FileStream = fileStream;
+        }
+
+        public string ContentType { get; }
+        public string ContentDisposition { get; }
+        public FileStream FileStream { get; }
+    }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,8 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-b89493e-dist-tests";
-
+        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";
@@ -110,7 +109,7 @@ namespace CodexPlugin
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
                 var account = marketplaceSetup.EthAccountSetup.GetNew();
-                AddEnvVar("PRIV_KEY", account.PrivateKey);
+                AddEnvVar("ETH_PRIVATE_KEY", account.PrivateKey);
                 Additional(account);
 
                 SetCommandOverride(marketplaceSetup);

ProjectPlugins/CodexPlugin/CodexLogLine.cs

@@ -48,6 +48,11 @@ namespace CodexPlugin
         public string Message { get; set; } = string.Empty;
         public Dictionary<string, string> Attributes { get; private set; } = new Dictionary<string, string>();
 
+        public override string ToString()
+        {
+            return Message;
+        }
+
         /// <summary>
         /// After too much time spent cursing at regexes, here's what I got:
         /// Parses input string into 'key=value' pair, considerate of quoted (") values.

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -1,4 +1,5 @@
-using Core;
+using CodexPlugin.Hooks;
+using Core;
 using FileUtils;
 using GethPlugin;
 using KubernetesWorkflow;
@@ -12,12 +13,24 @@ namespace CodexPlugin
     public interface ICodexNode : IHasContainer, IHasMetricsScrapeTarget, IHasEthAddress
     {
         string GetName();
-        DebugInfo GetDebugInfo();
+        string GetPeerId();
+        DebugInfo GetDebugInfo(bool log = false);
+        string GetSpr();
         DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
         ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
+        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
+        (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, string fileLabel = "");
+        (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
+        LocalDataset DownloadStreamless(ContentId cid);
+        /// <summary>
+        /// TODO: This will monitor the quota-used of the node until 'size' bytes are added. That's a very bad way
+        /// to track the streamless download progress. Replace it once we have a good API for this.
+        /// </summary>
+        LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size);
+        LocalDataset DownloadManifestOnly(ContentId cid);
         LocalDatasetList LocalFiles();
         CodexSpace Space();
         void ConnectToPeer(ICodexNode node);
@@ -39,19 +52,36 @@ namespace CodexPlugin
     public class CodexNode : ICodexNode
     {
         private const string UploadFailedMessage = "Unable to store block";
+        private readonly ILog log;
         private readonly IPluginTools tools;
+        private readonly ICodexNodeHooks hooks;
         private readonly EthAccount? ethAccount;
         private readonly TransferSpeeds transferSpeeds;
+        private string peerId = string.Empty;
+        private string nodeId = string.Empty;
 
-        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAccount? ethAccount)
+        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, ICodexNodeHooks hooks, EthAccount? ethAccount)
         {
             this.tools = tools;
             this.ethAccount = ethAccount;
             CodexAccess = codexAccess;
             Group = group;
             Marketplace = marketplaceAccess;
+            this.hooks = hooks;
             Version = new DebugInfoVersion();
             transferSpeeds = new TransferSpeeds();
+
+            log = new LogPrefixer(tools.GetLog(), $"{GetName()} ");
+        }
+
+        public void Awake()
+        {
+            hooks.OnNodeStarting(Container.Recipe.RecipeCreatedUtc, Container.Recipe.Image, ethAccount);
+        }
+
+        public void Initialize()
+        {
+            hooks.OnNodeStarted(peerId, nodeId);
         }
 
         public RunningPod Pod { get { return CodexAccess.Container; } }
@@ -95,14 +125,27 @@ namespace CodexPlugin
             return Container.Name;
         }
 
-        public DebugInfo GetDebugInfo()
+        public string GetPeerId()
+        {
+            return peerId;
+        }
+
+        public DebugInfo GetDebugInfo(bool log = false)
         {
             var debugInfo = CodexAccess.GetDebugInfo();
-            var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
-            Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
+            if (log)
+            {
+                var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
+                Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
+            }
             return debugInfo;
         }
 
+        public string GetSpr()
+        {
+            return CodexAccess.GetSpr();
+        }
+
         public DebugPeer GetDebugPeer(string peerId)
         {
             return CodexAccess.GetDebugPeer(peerId);
@@ -115,43 +158,93 @@ namespace CodexPlugin
 
         public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
         {
-            CodexAccess.LogDiskSpace("Before upload");
+            return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"", onFailure);
+        }
 
+        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
+        {
             using var fileStream = File.OpenRead(file.Filename);
+            var uniqueId = Guid.NewGuid().ToString();
+            var size = file.GetFilesize();
 
-            var logMessage = $"Uploading file {file.Describe()}...";
-            Log(logMessage);
-            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
+            hooks.OnFileUploading(uniqueId, size);
+
+            var input = new UploadInput(contentType, contentDisposition, fileStream);
+            var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
+            var measurement = Stopwatch.Measure(log, logMessage, () =>
             {
-                return CodexAccess.UploadFile(fileStream, onFailure);
+                return CodexAccess.UploadFile(input, onFailure);
             });
 
             var response = measurement.Value;
-            transferSpeeds.AddUploadSample(file.GetFilesize(), measurement.Duration);
+            transferSpeeds.AddUploadSample(size, measurement.Duration);
 
             if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
             if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
 
-            Log($"Uploaded file. Received contentId: '{response}'.");
-            CodexAccess.LogDiskSpace("After upload");
+            Log($"Uploaded file {file.Describe()}. Received contentId: '{response}'.");
 
-            return new ContentId(response);
+            var cid = new ContentId(response);
+            hooks.OnFileUploaded(uniqueId, size, cid);
+            return cid;
         }
 
         public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
         {
-            return DownloadContent(contentId, DoNothing, fileLabel);
+            return DownloadContentT(contentId, fileLabel).Item1;
         }
 
         public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
         {
-            var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
-            Log(logMessage);
+            return DownloadContentT(contentId, onFailure, fileLabel).Item1;
+        }
+
+        public (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, string fileLabel = "")
+        {
+            return DownloadContentT(contentId, DoNothing, fileLabel);
+        }
+
+        public (TrackedFile?, TimeSpan) DownloadContentT(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
+        {
             var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
-            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
-            transferSpeeds.AddDownloadSample(file.GetFilesize(), measurement);
-            Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
-            return file;
+            hooks.OnFileDownloading(contentId);
+            Log($"Downloading '{contentId}'...");
+
+            var logMessage = $"Downloaded '{contentId}' to '{file.Filename}'";
+            var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
+
+            var size = file.GetFilesize();
+            transferSpeeds.AddDownloadSample(size, measurement);
+            hooks.OnFileDownloaded(size, contentId);
+
+            return (file, measurement);
+        }
+
+        public LocalDataset DownloadStreamless(ContentId cid)
+        {
+            Log($"Downloading streamless '{cid}' (no-wait)");
+            return CodexAccess.DownloadStreamless(cid);
+        }
+
+        public LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size)
+        {
+            Log($"Downloading streamless '{cid}' (wait till finished)");
+
+            var sw = Stopwatch.Measure(log, nameof(DownloadStreamlessWait), () =>
+            {
+                var startSpace = Space();
+                var result = CodexAccess.DownloadStreamless(cid);
+                WaitUntilQuotaUsedIncreased(startSpace, size);
+                return result;
+            });
+
+            return sw.Value;
+        }
+
+        public LocalDataset DownloadManifestOnly(ContentId cid)
+        {
+            Log($"Downloading manifest-only '{cid}'");
+            return CodexAccess.DownloadManifestOnly(cid);
         }
 
         public LocalDatasetList LocalFiles()
@@ -188,6 +281,8 @@ namespace CodexPlugin
         public void Stop(bool waitTillStopped)
         {
             Log("Stopping...");
+            hooks.OnNodeStopping();
+
             CrashWatcher.Stop();
             Group.Stop(this, waitTillStopped);
         }
@@ -195,7 +290,8 @@ namespace CodexPlugin
         public void EnsureOnlineGetVersionResponse()
         {
             var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
-            var nodePeerId = debugInfo.Id;
+            peerId = debugInfo.Id;
+            nodeId = debugInfo.Table.LocalNode.NodeId;
             var nodeName = CodexAccess.Container.Name;
 
             if (!debugInfo.Version.IsValid())
@@ -203,12 +299,18 @@ namespace CodexPlugin
                 throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
             }
 
-            var log = tools.GetLog();
-            log.AddStringReplace(nodePeerId, nodeName);
+            log.AddStringReplace(peerId, nodeName);
+            log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName);
             log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName);
+            log.AddStringReplace(CodexUtils.ToShortId(debugInfo.Table.LocalNode.NodeId), nodeName);
             Version = debugInfo.Version;
         }
 
+        public override string ToString()
+        {
+            return $"CodexNode:{GetName()}";
+        }
+
         private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)
         {
             // The peer we want to connect is in a different pod.
@@ -223,21 +325,67 @@ namespace CodexPlugin
 
         private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
         {
-            CodexAccess.LogDiskSpace("Before download");
-
             using var fileStream = File.OpenWrite(file.Filename);
+            var timeout = tools.TimeSet.HttpCallTimeout();
             try
             {
-                using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
-                downloadStream.CopyTo(fileStream);
+                // Type of stream generated by openAPI client does not support timeouts.
+                var start = DateTime.UtcNow;
+                var cts = new CancellationTokenSource();
+                var downloadTask = Task.Run(() =>
+                {
+                    using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
+                    downloadStream.CopyTo(fileStream);
+                }, cts.Token);
+                
+                while (DateTime.UtcNow - start < timeout)
+                {
+                    if (downloadTask.IsFaulted) throw downloadTask.Exception;
+                    if (downloadTask.IsCompletedSuccessfully) return;
+                    Thread.Sleep(100);
+                }
+
+                cts.Cancel();
+                throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
             }
             catch
             {
                 Log($"Failed to download file '{contentId}'.");
                 throw;
             }
+        }
 
-            CodexAccess.LogDiskSpace("After download");
+        public void WaitUntilQuotaUsedIncreased(CodexSpace startSpace, ByteSize expectedIncreaseOfQuotaUsed)
+        {
+            WaitUntilQuotaUsedIncreased(startSpace, expectedIncreaseOfQuotaUsed, TimeSpan.FromMinutes(2));
+        }
+
+        public void WaitUntilQuotaUsedIncreased(
+            CodexSpace startSpace,
+            ByteSize expectedIncreaseOfQuotaUsed,
+            TimeSpan maxTimeout)
+        {
+            Log($"Waiting until quotaUsed " +
+                $"(start: {startSpace.QuotaUsedBytes}) " +
+                $"increases by {expectedIncreaseOfQuotaUsed} " +
+                $"to reach {startSpace.QuotaUsedBytes + expectedIncreaseOfQuotaUsed.SizeInBytes}");
+
+            var retry = new Retry($"Checking local space for quotaUsed increase of {expectedIncreaseOfQuotaUsed}",
+            maxTimeout: maxTimeout,
+            sleepAfterFail: TimeSpan.FromSeconds(3),
+            onFail: f => { });
+
+            retry.Run(() =>
+            {
+                var space = Space();
+                var increase = space.QuotaUsedBytes - startSpace.QuotaUsedBytes;
+
+                if (increase < expectedIncreaseOfQuotaUsed.SizeInBytes)
+                    throw new Exception($"Expected quota-used not reached. " +
+                        $"Expected increase: {expectedIncreaseOfQuotaUsed.SizeInBytes} " +
+                        $"Actual increase: {increase} " +
+                        $"Actual used: {space.QuotaUsedBytes}");
+            });
         }
 
         private void EnsureMarketplace()
@@ -247,7 +395,7 @@ namespace CodexPlugin
 
         private void Log(string msg)
         {
-            tools.GetLog().Log($"{GetName()}: {msg}");
+            log.Log(msg);
         }
 
         private void DoNothing(Failure failure)

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -1,4 +1,5 @@
-using Core;
+using CodexPlugin.Hooks;
+using Core;
 using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
@@ -14,23 +15,27 @@ namespace CodexPlugin
     public class CodexNodeFactory : ICodexNodeFactory
     {
         private readonly IPluginTools tools;
+        private readonly CodexHooksFactory codexHooksFactory;
 
-        public CodexNodeFactory(IPluginTools tools)
+        public CodexNodeFactory(IPluginTools tools, CodexHooksFactory codexHooksFactory)
         {
             this.tools = tools;
+            this.codexHooksFactory = codexHooksFactory;
         }
 
         public CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
         {
             var ethAccount = GetEthAccount(access);
-            var marketplaceAccess = GetMarketplaceAccess(access, ethAccount);
-            return new CodexNode(tools, access, group, marketplaceAccess, ethAccount);
+            var hooks = codexHooksFactory.CreateHooks(access.Container.Name);
+
+            var marketplaceAccess = GetMarketplaceAccess(access, ethAccount, hooks);
+            return new CodexNode(tools, access, group, marketplaceAccess, hooks, ethAccount);
         }
 
-        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAccount? ethAccount)
+        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAccount? ethAccount, ICodexNodeHooks hooks)
         {
             if (ethAccount == null) return new MarketplaceUnavailable();
-            return new MarketplaceAccess(tools.GetLog(), codexAccess);
+            return new MarketplaceAccess(tools.GetLog(), codexAccess, hooks);
         }
 
         private EthAccount? GetEthAccount(CodexAccess access)

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -79,13 +79,16 @@ namespace CodexPlugin
             }
 
             Version = first;
+            foreach (var node in Nodes) node.Initialize();
         }
 
         private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory)
         {
             var watcher = factory.CreateCrashWatcher(c.Containers.Single());
             var access = new CodexAccess(tools, c, watcher);
-            return factory.CreateOnlineCodexNode(access, this);
+            var node = factory.CreateOnlineCodexNode(access, this);
+            node.Awake();
+            return node;
         }
     }
 }

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -1,3 +1,4 @@
+using CodexPlugin.Hooks;
 using Core;
 using KubernetesWorkflow.Types;
 
@@ -19,7 +20,7 @@ namespace CodexPlugin
 
         public void Announce()
         {
-            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
+            Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
         }
 
         public void AddMetadata(IAddMetadata metadata)
@@ -54,9 +55,18 @@ namespace CodexPlugin
             {
                 mconfig.GethNode.SendEth(node, mconfig.MarketplaceSetup.InitialEth);
                 mconfig.CodexContracts.MintTestTokens(node, mconfig.MarketplaceSetup.InitialTestTokens);
+
+                Log($"Send {mconfig.MarketplaceSetup.InitialEth} and " +
+                    $"minted {mconfig.MarketplaceSetup.InitialTestTokens} for " +
+                    $"{node.GetName()} (address: {node.EthAddress})");
             }
         }
 
+        public void SetCodexHooksProvider(ICodexHooksProvider hooksProvider)
+        {
+            codexStarter.HooksFactory.Provider = hooksProvider;
+        }
+
         private CodexSetup GetSetup(int numberOfNodes, Action<ICodexSetup> setup)
         {
             var codexSetup = new CodexSetup(numberOfNodes);
@@ -64,5 +74,10 @@ namespace CodexPlugin
             setup(codexSetup);
             return codexSetup;
         }
+
+        private void Log(string msg)
+        {
+            tools.GetLog().Log(msg);
+        }
     }
 }

ProjectPlugins/CodexPlugin/CodexPlugin.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
@@ -29,6 +29,7 @@
   <ItemGroup>
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
     <ProjectReference Include="..\..\Framework\KubernetesWorkflow\KubernetesWorkflow.csproj" />
+    <ProjectReference Include="..\..\Framework\OverwatchTranscript\OverwatchTranscript.csproj" />
     <ProjectReference Include="..\CodexContractsPlugin\CodexContractsPlugin.csproj" />
     <ProjectReference Include="..\GethPlugin\GethPlugin.csproj" />
     <ProjectReference Include="..\MetricsPlugin\MetricsPlugin.csproj" />