Fixes too-frequent save of overview json. Resets failure count on upload.

Adds proofSubmitted event
Adds proof-period report to chain-events.
2025-03-04 16:14:29 +01:00 · 2025-03-04 15:58:45 +01:00 · 2025-03-04 15:24:25 +01:00 · 2025-03-04 10:33:22 +01:00 · 2025-03-04 10:03:55 +01:00 · 2025-03-03 18:56:18 +01:00
400 changed files with 18021 additions and 4074 deletions
@@ -0,0 +1,30 @@
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md
+!**/.gitignore
+!.git/HEAD
+!.git/config
+!.git/packed-refs
+!.git/refs/heads/**
@@ -0,0 +1,10 @@
+# Set default behavior to automatically normalize line endings.
+* text=auto
+
+# Force bash scripts to always use lf line endings so that if a repo is accessed
+# in Unix via a file share from Windows, the scripts will work.
+*.sh text eol=lf
+
+# Likewise, force cmd and batch scripts to always use crlf
+*.cmd text eol=crlf
+*.bat text eol=crlf
@@ -0,0 +1,27 @@
+name: Docker - AutoClient
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/AutoClient/**'
+      - '!Tools/AutoClient/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-autoclient.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
+    with:
+      docker_file: Tools/AutoClient/docker/Dockerfile
+      dockerhub_repo: codexstorage/codex-autoclient
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
+    secrets: inherit
@@ -13,16 +13,15 @@ on:
      - 'Framework/**'
      - 'ProjectPlugins/**'
      - .github/workflows/docker-discordbot.yml
-      - .github/workflows/docker-reusable.yml
  workflow_dispatch:


 jobs:
  build-and-push:
    name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
    with:
      docker_file: Tools/BiblioTech/docker/Dockerfile
-      docker_repo: codexstorage/codex-discordbot
+      dockerhub_repo: codexstorage/codex-discordbot
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
    secrets: inherit
-
@@ -0,0 +1,26 @@
+name: Docker - KeyMaker
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/KeyMaker/**'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-keymaker.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
+    with:
+      docker_file: Tools/KeyMaker/docker/Dockerfile
+      dockerhub_repo: codexstorage/codex-keymaker
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
+    secrets: inherit
@@ -0,0 +1,26 @@
+name: Docker - MarketInsights API
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/MarketInsights/**'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-marketinsights.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
+    with:
+      docker_file: Tools/MarketInsights/Dockerfile
+      dockerhub_repo: codexstorage/codex-marketinsights
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
+    secrets: inherit
@@ -1,178 +0,0 @@
-name: Reusable - Docker
-
-
-on:
-  workflow_call:
-    inputs:
-      docker_file:
-        default: docker/Dockerfile
-        description: Dockerfile
-        required: false
-        type: string
-      docker_repo:
-        default: codexstorage/cs-codex-dist-tests
-        description: DockerHub repository
-        required: false
-        type: string
-      tag_latest:
-        default: true
-        description: Set latest tag for Docker images
-        required: false
-        type: boolean
-      tag_sha:
-        default: true
-        description: Set Git short commit as Docker tag
-        required: false
-        type: boolean
-      tag_suffix:
-        default: ''
-        description: Suffix for Docker images tag
-        required: false
-        type: string
-
-
-env:
-  DOCKER_FILE: ${{ inputs.docker_file }}
-  DOCKER_REPO: ${{ inputs.docker_repo }}
-  TAG_LATEST: ${{ inputs.tag_latest }}
-  TAG_SHA: ${{ inputs.tag_sha }}
-  TAG_SUFFIX: ${{ inputs.tag_suffix }}
-
-
-jobs:
-  # Build platform specific image
-  build:
-    strategy:
-      fail-fast: true
-      matrix:
-        target:
-          - os: linux
-            arch: amd64
-          - os: linux
-            arch: arm64
-        include:
-          - target:
-              os: linux
-              arch: amd64
-            builder: ubuntu-22.04
-          - target:
-              os: linux
-              arch: arm64
-            builder: buildjet-4vcpu-ubuntu-2204-arm
-
-    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
-    runs-on: ${{ matrix.builder }}
-    env:
-      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.DOCKER_REPO }}
-
-      - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Build and Push by digest
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ${{ env.DOCKER_FILE }}
-          platforms: ${{ env.PLATFORM }}
-          push: true
-          labels: ${{ steps.meta.outputs.labels }}
-          outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
-
-      - name: Docker - Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-      - name: Docker - Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: digests-${{ matrix.target.arch }}
-          path: /tmp/digests
-          if-no-files-found: error
-          retention-days: 1
-
-
-  # Publish multi-platform image
-  publish:
-    name: Publish multi-platform image
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Docker - Variables
-        run: |
-          # Adjust custom suffix when set and
-          if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
-            echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
-          fi
-          # Disable SHA tags on tagged release
-          if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
-            echo "TAG_SHA=false" >>$GITHUB_ENV
-          fi
-          # Handle latest and latest-custom using raw
-          if [[ ${{ env.TAG_SHA }} == "false" ]]; then
-            echo "TAG_LATEST=false" >>$GITHUB_ENV
-            echo "TAG_RAW=true" >>$GITHUB_ENV
-            if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
-              echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
-            else
-              echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
-            fi
-          else
-            echo "TAG_RAW=false" >>$GITHUB_ENV
-          fi
-
-      - name: Docker - Download digests
-        uses: actions/download-artifact@v4
-        with:
-          pattern: digests-*
-          merge-multiple: true
-          path: /tmp/digests
-
-      - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.DOCKER_REPO }}
-          flavor: |
-            latest=${{ env.TAG_LATEST }}
-            suffix=${{ env.TAG_SUFFIX }},onlatest=true
-          tags: |
-            type=semver,pattern={{version}}
-            type=raw,enable=${{ env.TAG_RAW }},value=latest
-            type=sha,enable=${{ env.TAG_SHA }}
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Create manifest list and push
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
-
-      - name: Docker - Inspect image
-        run: |
-          docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}
@@ -1,5 +1,6 @@
 name: Docker - Rewarder Bot

+
 on:
  push:
    branches:
@@ -12,15 +13,15 @@ on:
      - 'Framework/**'
      - 'ProjectPlugins/**'
      - .github/workflows/docker-rewarder.yml
-      - .github/workflows/docker-reusable.yml
  workflow_dispatch:

+
 jobs:
  build-and-push:
    name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
    with:
      docker_file: Tools/TestNetRewarder/docker/Dockerfile
-      docker_repo: codexstorage/codex-rewarderbot
+      dockerhub_repo: codexstorage/codex-rewarderbot
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
    secrets: inherit
-
@@ -11,12 +11,15 @@ on:
      - docker/Dockerfile
      - docker/docker-entrypoint.sh
      - .github/workflows/docker-runner.yml
-      - .github/workflows/docker-reusable.yml
  workflow_dispatch:


 jobs:
  build-and-push:
    name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
+    with:
+      docker_file: docker/Dockerfile
+      dockerhub_repo: codexstorage/cs-codex-dist-tests
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
    secrets: inherit
@@ -80,7 +80,7 @@ env:
  TESTS_TARGET_DURATION: 2d
  TESTS_FILTER: ""
  TESTS_CLEANUP: true
-  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  JOB_MANIFEST: docker/job-continuous-tests.yaml
  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
  KUBE_VERSION: v1.28.2

@@ -146,7 +146,7 @@ jobs:
        if: false
        run: |
          # Variables
-          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to run a node
          duration=600
          namespace="${{ env.NAMESPACE }}"
          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")
@@ -32,7 +32,7 @@ env:
  NAMEPREFIX: d-tests-runner
  NAMESPACE: default
  COMMAND: dotnet test Tests/CodexTests
-  JOB_MANIFEST: docker/dist-tests-job.yaml
+  JOB_MANIFEST: docker/job-dist-tests.yaml
  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
  KUBE_VERSION: v1.28.2

@@ -0,0 +1,122 @@
+name: Run Release Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+      workflow_source:
+        description: Workflow source
+        required: false
+        type: string
+        default: ''
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
+  TEST_TYPE: release-tests
+  NAMEPREFIX: r-tests
+  NAMESPACE: default
+  JOB_MANIFEST: docker/job-release-tests.yaml
+  COMMAND: dotnet test Tests/CodexReleaseTests
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.30.5
+
+
+jobs:
+  run_tests:
+    name: Run Release Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
+          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'${COMMAND}'"')" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job to run tests
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}"
+          echo "Runner pod: $(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}')"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+
+      - name: Show Runner logs
+        run: |
+          # Variables
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          namespace="${{ env.NAMESPACE }}"
+          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath='{.items[0].metadata.name}')
+
+          # Check Pod status
+          WAIT=120
+          SECONDS=0
+          sleep=1
+          while (( SECONDS < WAIT )); do
+            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}')
+            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in the $phase state - Get the logs"; break; } || { echo "Pod $pod is in the $phase state - Retry in $sleep second(s) / $((WAIT - SECONDS))"; }
+            sleep $sleep
+          done
+
+          # Get logs
+          while [[ $(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}') == "Running" ]]; do
+            echo "Show ${pod} logs ..."
+            echo "----"
+            kubectl logs $pod -n $namespace -f || true
+            sleep 1
+          done
+
+      - name: Set workflow status from job status
+        run: |
+          sleep 5
+          job_status=$(kubectl get jobs ${{ env.NAMEPREFIX }} -n ${{ env.NAMESPACE }} -o jsonpath='{.status.conditions[0].type}')
+          echo "Job status: $job_status"
+          if [[ "${job_status}" != "Complete" ]]; then exit 1; fi
@@ -1,4 +1,6 @@
 .vs
 obj
 bin
-.vscode
+.vscode
+Tools/AutoClient/datapath
+.editorconfig
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -1,4 +1,6 @@
-using System.Reflection;
+using System.Globalization;
+using System.Numerics;
+using System.Reflection;

 namespace ArgsUniform
 {
@@ -103,8 +105,12 @@ namespace ArgsUniform
                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(ulong)) return AssignUlong(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(BigInteger)) return AssignBigInt(result, uniformProperty, value);

-                    throw new NotSupportedException();
+                    throw new NotSupportedException(
+                        $"Unsupported property type '${uniformProperty.PropertyType}' " +
+                        $"for property '${uniformProperty.Name}'.");
                }
            }
        }
@@ -122,7 +128,27 @@ namespace ArgsUniform

        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
        {
-            if (int.TryParse(value.ToString(), out int i))
+            if (int.TryParse(value.ToString(), CultureInfo.InvariantCulture, out int i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignUlong(T? result, PropertyInfo uniformProperty, object value)
+        {
+            if (ulong.TryParse(value.ToString(), CultureInfo.InvariantCulture, out ulong i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignBigInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (BigInteger.TryParse(value.ToString(), CultureInfo.InvariantCulture, out BigInteger i))
            {
                uniformProperty.SetValue(result, i);
                return true;
@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
    public class BlockCache
    {
@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
    public class BlockTimeEntry
    {
@@ -1,6 +1,6 @@
 using Logging;

-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
    public class BlockTimeFinder
    {
@@ -29,7 +29,8 @@ namespace NethereumWorkflow.BlockUtils
        public ulong? GetHighestBlockNumberBefore(DateTime moment)
        {
            bounds.Initialize();
-            if (moment <= bounds.Genesis.Utc) return null;
+            if (moment < bounds.Genesis.Utc) return null;
+            if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
            if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;

            return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
@@ -38,7 +39,8 @@ namespace NethereumWorkflow.BlockUtils
        public ulong? GetLowestBlockNumberAfter(DateTime moment)
        {
            bounds.Initialize();
-            if (moment >= bounds.Current.Utc) return null;
+            if (moment > bounds.Current.Utc) return null;
+            if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber;
            if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;

            return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;
@@ -46,7 +48,7 @@ namespace NethereumWorkflow.BlockUtils

        private ulong Log(Func<ulong> operation)
        {
-            var sw = Stopwatch.Begin(log, nameof(BlockTimeFinder));
+            var sw = Stopwatch.Begin(log, nameof(BlockTimeFinder), true);
            var result = operation();
            sw.End($"(Bounds: [{bounds.Genesis.BlockNumber}-{bounds.Current.BlockNumber}] Cache: {cache.Size})");

@@ -1,5 +1,11 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
+    public interface IWeb3Blocks
+    {
+        ulong GetCurrentBlockNumber();
+        DateTime? GetTimestampForBlock(ulong blockNumber);
+    }
+
    public class BlockchainBounds
    {
        private readonly BlockCache cache;
@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+	<ItemGroup>
+		<PackageReference Include="Nethereum.Web3" Version="4.14.0" />
+	</ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>
@@ -1,7 +1,7 @@
 using Nethereum.Hex.HexTypes;
 using System.Numerics;

-namespace NethereumWorkflow
+namespace BlockchainUtils
 {
    public static class ConversionExtensions
    {
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -9,6 +9,7 @@
  <ItemGroup>
    <ProjectReference Include="..\FileUtils\FileUtils.csproj" />
    <ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
+    <ProjectReference Include="..\WebUtils\WebUtils.csproj" />
  </ItemGroup>

 </Project>
@@ -1,5 +1,6 @@
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
+using Logging;

 namespace Core
 {
@@ -30,11 +31,7 @@ namespace Core
        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
        {
            var workflow = entryPoint.Tools.CreateWorkflow();
-            var file = entryPoint.Tools.GetLog().CreateSubfile();
-            entryPoint.Tools.GetLog().Log($"Downloading container log for '{container.Name}' to file '{file.FullFilename}'...");
-            var logHandler = new LogDownloadHandler(container.Name, file);
-            workflow.DownloadContainerLog(container, logHandler, tailLines);
-            return logHandler.DownloadLog();
+            return workflow.DownloadContainerLog(container, tailLines);
        }

        public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)
@@ -1,79 +0,0 @@
-using Logging;
-
-namespace Core
-{
-    public interface IDownloadedLog
-    {
-        void IterateLines(Action<string> action);
-        string[] GetLinesContaining(string expectedString);
-        string[] FindLinesThatContain(params string[] tags);
-        void DeleteFile();
-    }
-
-    internal class DownloadedLog : IDownloadedLog
-    {
-        private readonly LogFile logFile;
-
-        internal DownloadedLog(LogFile logFile)
-        {
-            this.logFile = logFile;
-        }
-
-        public void IterateLines(Action<string> action)
-        {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                action(line);
-                line = streamReader.ReadLine();
-            }
-        }
-
-        public string[] GetLinesContaining(string expectedString)
-        {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-            var lines = new List<string>();
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (line.Contains(expectedString))
-                {
-                    lines.Add(line);
-                }
-                line = streamReader.ReadLine();
-            }
-
-            return lines.ToArray(); ;
-        }
-
-        public string[] FindLinesThatContain(params string[] tags)
-        {
-            var result = new List<string>();
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (tags.All(line.Contains))
-                {
-                    result.Add(line);
-                }
-
-                line = streamReader.ReadLine();
-            }
-
-            return result.ToArray();
-        }
-
-        public void DeleteFile()
-        {
-            File.Delete(logFile.FullFilename);
-        }
-    }
-}
@@ -1,5 +1,6 @@
 using KubernetesWorkflow;
 using Logging;
+using WebUtils;

 namespace Core
 {
@@ -8,16 +9,16 @@ namespace Core
        private readonly IToolsFactory toolsFactory;
        private readonly PluginManager manager = new PluginManager();

-        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
        {
-            toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, timeSet);
+            toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, webCallTimeSet, k8STimeSet);

            Tools = toolsFactory.CreateTools();
            manager.InstantiatePlugins(PluginFinder.GetPluginTypes(), toolsFactory);
        }

        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder)
-            : this(log, configuration, fileManagerRootFolder, new DefaultTimeSet())
+            : this(log, configuration, fileManagerRootFolder, new DefaultWebCallTimeSet(), new DefaultK8sTimeSet())
        {
        }

@@ -38,10 +39,14 @@ namespace Core
            return new CoreInterface(this);
        }

-        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
        {
-            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles);
-            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
+            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
        }

        internal T GetPlugin<T>() where T : IProjectPlugin
@@ -1,28 +0,0 @@
-using KubernetesWorkflow;
-using Logging;
-
-namespace Core
-{
-    internal class LogDownloadHandler : LogHandler, ILogHandler
-    {
-        private readonly LogFile log;
-
-        internal LogDownloadHandler(string description, LogFile log)
-        {
-            this.log = log;
-
-            log.Write($"{description} -->> {log.FullFilename}");
-            log.WriteRaw(description);
-        }
-
-        internal IDownloadedLog DownloadLog()
-        {
-            return new DownloadedLog(log);
-        }
-
-        protected override void ProcessLine(string line)
-        {
-            log.WriteRaw(line);
-        }
-    }
-}
@@ -34,12 +34,12 @@
            return metadata;
        }

-        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
        {
            foreach (var pair in pairs)
            {
                pair.Plugin.Decommission();
-                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles);
+                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
            }
        }

@@ -1,12 +1,20 @@
 using FileUtils;
 using KubernetesWorkflow;
 using Logging;
+using WebUtils;

 namespace Core
 {
-    public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
+    public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactory, IFileTool
    {
-        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles);
+        IWebCallTimeSet WebCallTimeSet { get; }
+        IK8sTimeSet K8STimeSet { get; }
+
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone);
    }

    public interface IWorkflowTool
@@ -19,13 +27,6 @@ namespace Core
        ILog GetLog();
    }

-    public interface IHttpFactoryTool
-    {
-        IHttp CreateHttp(Action<HttpClient> onClientCreated);
-        IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
-        IHttp CreateHttp();
-    }
-
    public interface IFileTool
    {
        IFileManager GetFileManager();
@@ -33,37 +34,42 @@ namespace Core

    internal class PluginTools : IPluginTools
    {
-        private readonly ITimeSet timeSet;
        private readonly WorkflowCreator workflowCreator;
+        private readonly HttpFactory httpFactory;
        private readonly IFileManager fileManager;
        private readonly LogPrefixer log;

-        internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet)
+        internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
        {
            this.log = new LogPrefixer(log);
            this.workflowCreator = workflowCreator;
-            this.timeSet = timeSet;
+            httpFactory = new HttpFactory(log, webCallTimeSet);
+            WebCallTimeSet = webCallTimeSet;
+            K8STimeSet = k8STimeSet;
            fileManager = new FileManager(log, fileManagerRootFolder);
        }

+        public IWebCallTimeSet WebCallTimeSet { get; }
+        public IK8sTimeSet K8STimeSet { get; }
+
        public void ApplyLogPrefix(string prefix)
        {
            log.Prefix = prefix;
        }

-        public IHttp CreateHttp(Action<HttpClient> onClientCreated)
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
        {
-            return CreateHttp(onClientCreated, timeSet);
+            return httpFactory.CreateHttp(id, onClientCreated);
        }

-        public IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet ts)
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet timeSet)
        {
-            return new Http(log, ts, onClientCreated);
+            return httpFactory.CreateHttp(id, onClientCreated, timeSet);
        }

-        public IHttp CreateHttp()
+        public IHttp CreateHttp(string id)
        {
-            return new Http(log, timeSet);
+            return httpFactory.CreateHttp(id);
        }

        public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)
@@ -71,9 +77,9 @@ namespace Core
            return workflowCreator.CreateWorkflow(namespaceOverride);
        }

-        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles)
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
        {
-            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace();
+            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace(waitTillDone);
            if (deleteTrackedFiles) fileManager.DeleteAllFiles();
        }

@@ -1,5 +1,6 @@
 using KubernetesWorkflow;
 using Logging;
+using WebUtils;

 namespace Core
 {
@@ -13,19 +14,21 @@ namespace Core
        private readonly ILog log;
        private readonly WorkflowCreator workflowCreator;
        private readonly string fileManagerRootFolder;
-        private readonly ITimeSet timeSet;
+        private readonly IWebCallTimeSet webCallTimeSet;
+        private readonly IK8sTimeSet k8STimeSet;

-        public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
        {
            this.log = log;
            workflowCreator = new WorkflowCreator(log, configuration);
            this.fileManagerRootFolder = fileManagerRootFolder;
-            this.timeSet = timeSet;
+            this.webCallTimeSet = webCallTimeSet;
+            this.k8STimeSet = k8STimeSet;
        }

        public PluginTools CreateTools()
        {
-            return new PluginTools(log, workflowCreator, fileManagerRootFolder, timeSet);
+            return new PluginTools(log, workflowCreator, fileManagerRootFolder, webCallTimeSet, k8STimeSet);
        }
    }
 }
@@ -13,9 +13,9 @@ namespace DiscordRewards
    public enum CheckType
    {
        Uninitialized,
-        FilledSlot,
-        FinishedSlot,
-        PostedContract,
-        StartedContract,
+        HostFilledSlot,
+        HostFinishedSlot,
+        ClientPostedContract,
+        ClientStartedContract,
    }
 }
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -3,8 +3,13 @@
    public class GiveRewardsCommand
    {
        public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
-        public MarketAverage[] Averages { get; set; } = Array.Empty<MarketAverage>();
-        public string[] EventsOverview { get; set; } = Array.Empty<string>();
+        public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty<ChainEventMessage>();
+        public string[] Errors { get; set; } = Array.Empty<string>();
+
+        public bool HasAny()
+        {
+            return Rewards.Any() || EventsOverview.Any();
+        }
    }

    public class RewardUsersCommand
@@ -13,14 +18,9 @@
        public string[] UserAddresses { get; set; } = Array.Empty<string>();
    }

-    public class MarketAverage
+    public class ChainEventMessage
    {
-        public int NumberOfFinished { get; set; }
-        public int TimeRangeSeconds { get; set; }
-        public float Price { get; set; }
-        public float Size { get; set; }
-        public float Duration { get; set; }
-        public float Collateral { get; set; }
-        public float ProofProbability { get; set; }
+        public ulong BlockNumber { get; set; }
+        public string Message { get; set; } = string.Empty;
    }
 }
@@ -1,53 +1,53 @@
-using Utils;
-
-namespace DiscordRewards
+namespace DiscordRewards
 {
    public class RewardRepo
    {
        private static string Tag => RewardConfig.UsernameTag;

-        public RewardConfig[] Rewards { get; } = new RewardConfig[]
-        {
-            // Filled any slot
-            new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
-            {
-                Type = CheckType.FilledSlot
-            }),
+        public RewardConfig[] Rewards { get; } = new RewardConfig[0];

-            // Finished any slot
-            new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
-            {
-                Type = CheckType.FinishedSlot
-            }),
+        // Example configuration, from test server:
+        //{
+        //    // Filled any slot
+        //    new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFilledSlot
+        //    }),

-            // Finished a sizable slot
-            new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig
-            {
-                Type = CheckType.FinishedSlot,
-                MinSlotSize = 10.MB(),
-                MinDuration = TimeSpan.FromMinutes(5.0),
-            }),
+        //    // Finished any slot
+        //    new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFinishedSlot
+        //    }),

-            // Posted any contract
-            new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
-            {
-                Type = CheckType.PostedContract
-            }),
+        //    // Finished a sizable slot
+        //    new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFinishedSlot,
+        //        MinSlotSize = 10.MB(),
+        //        MinDuration = TimeSpan.FromMinutes(5.0),
+        //    }),

-            // Started any contract
-            new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
-            {
-                Type = CheckType.StartedContract
-            }),
+        //    // Posted any contract
+        //    new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientPostedContract
+        //    }),

-            // Started a sizable contract
-            new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig
-            {
-                Type = CheckType.StartedContract,
-                MinNumberOfHosts = 4,
-                MinSlotSize = 10.MB(),
-                MinDuration = TimeSpan.FromMinutes(5.0),
-            })
-        };
+        //    // Started any contract
+        //    new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientStartedContract
+        //    }),
+
+        //    // Started a sizable contract
+        //    new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientStartedContract,
+        //        MinNumberOfHosts = 4,
+        //        MinSlotSize = 10.MB(),
+        //        MinDuration = TimeSpan.FromMinutes(5.0),
+        //    })
+        //};
    }
 }
@@ -7,27 +7,35 @@ namespace FileUtils
    {
        TrackedFile CreateEmptyFile(string label = "");
        TrackedFile GenerateFile(ByteSize size, string label = "");
+        TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "");
        void DeleteAllFiles();
        void ScopedFiles(Action action);
        T ScopedFiles<T>(Func<T> action);
    }

+    public interface IGenerateOption
+    {
+        IGenerateOption Random(ByteSize size);
+        IGenerateOption StringRepeat(string str, ByteSize size);
+        IGenerateOption StringRepeat(string str, int times);
+        IGenerateOption ByteRepeat(byte[] bytes, ByteSize size);
+        IGenerateOption ByteRepeat(byte[] bytes, int times);
+    }
+
    public class FileManager : IFileManager
    {
-        public const int ChunkSize = 1024 * 1024 * 100;
-        private static NumberSource folderNumberSource = new NumberSource(0);
-        private readonly Random random = new Random();
+        private static readonly NumberSource folderNumberSource = new NumberSource(0);
        private readonly ILog log;
-        private readonly string rootFolder;
        private readonly string folder;
        private readonly List<List<TrackedFile>> fileSetStack = new List<List<TrackedFile>>();

+        public const int ChunkSize = 1024 * 1024 * 100;
+
        public FileManager(ILog log, string rootFolder)
        {
            folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));

            this.log = log;
-            this.rootFolder = rootFolder;
        }

        public TrackedFile CreateEmptyFile(string label = "")
@@ -41,10 +49,15 @@ namespace FileUtils
            return result;
        }

-        public TrackedFile GenerateFile(ByteSize size, string label)
+        public TrackedFile GenerateFile(ByteSize size, string label = "")
+        {
+            return GenerateFile(o => o.Random(size), label);
+        }
+
+        public TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "")
        {
            var sw = Stopwatch.Begin(log);
-            var result = GenerateRandomFile(size, label);
+            var result = RunGenerators(options, label);
            sw.End($"Generated file {result.Describe()}.");
            return result;
        }
@@ -57,16 +70,27 @@ namespace FileUtils
        public void ScopedFiles(Action action)
        {
            PushFileSet();
-            action();
-            PopFileSet();
+            try
+            {
+                action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
        }

        public T ScopedFiles<T>(Func<T> action)
        {
            PushFileSet();
-            var result = action();
-            PopFileSet();
-            return result;
+            try
+            {
+                return action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
        }

        private void PushFileSet()
@@ -89,26 +113,35 @@ namespace FileUtils
            if (!Directory.GetFiles(folder).Any()) DeleteDirectory();
        }

-        private TrackedFile GenerateRandomFile(ByteSize size, string label)
+        private TrackedFile RunGenerators(Action<IGenerateOption> options, string label)
        {
            var result = CreateEmptyFile(label);
-            CheckSpaceAvailable(result, size);
+            var generators = GetGenerators(options);
+            CheckSpaceAvailable(result, generators.GetRequiredSpace());

-            GenerateFileBytes(result, size);
+            using var stream = new FileStream(result.Filename, FileMode.Append);
+            generators.Run(stream);
            return result;
        }

-        private void CheckSpaceAvailable(TrackedFile testFile, ByteSize size)
+        private GeneratorCollection GetGenerators(Action<IGenerateOption> options)
+        {
+            var result = new GeneratorCollection();
+            options(result);
+            return result;
+        }
+
+        private void CheckSpaceAvailable(TrackedFile testFile, long requiredSize)
        {
            var file = new FileInfo(testFile.Filename);
            var drive = new DriveInfo(file.Directory!.Root.FullName);

            var spaceAvailable = drive.TotalFreeSpace;

-            if (spaceAvailable < size.SizeInBytes)
+            if (spaceAvailable < requiredSize)
            {
                var msg = $"Not enough disk space. " +
-                    $"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
+                    $"{Formatter.FormatByteSize(requiredSize)} required. " +
                    $"{Formatter.FormatByteSize(spaceAvailable)} available.";

                log.Log(msg);
@@ -116,34 +149,6 @@ namespace FileUtils
            }
        }

-        private void GenerateFileBytes(TrackedFile result, ByteSize size)
-        {
-            long bytesLeft = size.SizeInBytes;
-            int chunkSize = ChunkSize;
-            while (bytesLeft > 0)
-            {
-                try
-                {
-                    var length = Math.Min(bytesLeft, chunkSize);
-                    AppendRandomBytesToFile(result, length);
-                    bytesLeft -= length;
-                }
-                catch
-                {
-                    chunkSize = chunkSize / 2;
-                    if (chunkSize < 1024) throw;
-                }
-            }
-        }
-
-        private void AppendRandomBytesToFile(TrackedFile result, long length)
-        {
-            var bytes = new byte[length];
-            random.NextBytes(bytes);
-            using var stream = new FileStream(result.Filename, FileMode.Append);
-            stream.Write(bytes, 0, bytes.Length);
-        }
-
        private void EnsureDirectory()
        {
            Directory.CreateDirectory(folder);
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -0,0 +1,141 @@
+using System.Text;
+using Utils;
+
+namespace FileUtils
+{
+    public class GeneratorCollection : IGenerateOption
+    {
+        private readonly List<IGenerator> generators = new List<IGenerator>();
+
+        public IGenerateOption ByteRepeat(byte[] bytes, ByteSize size)
+        {
+            var times = size.SizeInBytes / bytes.Length;
+            generators.Add(new ByteRepeater(bytes, times));
+            return this;
+        }
+
+        public IGenerateOption ByteRepeat(byte[] bytes, int times)
+        {
+            generators.Add(new ByteRepeater(bytes, times));
+            return this;
+        }
+
+        public IGenerateOption Random(ByteSize size)
+        {
+            generators.Add(new RandomGenerator(size));
+            return this;
+        }
+
+        public IGenerateOption StringRepeat(string str, ByteSize size)
+        {
+            var times = size.SizeInBytes / str.Length;
+            generators.Add(new StringRepeater(str, times));
+            return this;
+        }
+
+        public IGenerateOption StringRepeat(string str, int times)
+        {
+            generators.Add(new StringRepeater(str, times));
+            return this;
+        }
+
+        public void Run(FileStream file)
+        {
+            foreach (var generator in generators)
+            {
+                generator.Generate(file);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return generators.Sum(g => g.GetRequiredSpace());
+        }
+    }
+
+    public interface IGenerator
+    {
+        void Generate(FileStream file);
+        long GetRequiredSpace();
+    }
+
+    public class ByteRepeater : IGenerator
+    {
+        private readonly byte[] bytes;
+        private readonly long times;
+
+        public ByteRepeater(byte[] bytes, long times)
+        {
+            this.bytes = bytes;
+            this.times = times;
+        }
+
+        public void Generate(FileStream file)
+        {
+            for (var i = 0; i < times; i++)
+            {
+                file.Write(bytes, 0, bytes.Length);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return bytes.Length * times;
+        }
+    }
+
+    public class StringRepeater : IGenerator
+    {
+        private readonly string str;
+        private readonly long times;
+
+        public StringRepeater(string str, long times)
+        {
+            this.str = str;
+            this.times = times;
+        }
+
+        public void Generate(FileStream file)
+        {
+            using var writer = new StreamWriter(file);
+            for (var i = 0; i < times; i++)
+            {
+                writer.Write(str);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return Encoding.ASCII.GetBytes(str).Length * times;
+        }
+    }
+
+    public class RandomGenerator : IGenerator
+    {
+        private readonly Random random = new Random();
+        private readonly ByteSize size;
+
+        public RandomGenerator(ByteSize size)
+        {
+            this.size = size;
+        }
+
+        public void Generate(FileStream file)
+        {
+            var bytesLeft = size.SizeInBytes;
+            while (bytesLeft > 0)
+            {
+                var size = Math.Min(bytesLeft, FileManager.ChunkSize);
+                var bytes = new byte[size];
+                random.NextBytes(bytes);
+                file.Write(bytes, 0, bytes.Length);
+                bytesLeft -= size;
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return size.SizeInBytes;
+        }
+    }
+}
@@ -14,6 +14,12 @@ namespace FileUtils
            Label = label;
        }

+        public static TrackedFile FromPath(ILog log, string filepath)
+        {
+            // todo: I don't wanne have to do this to call upload.
+            return new TrackedFile(log, filepath, string.Empty);
+        }
+
        public string Filename { get; }
        public string Label { get; }

@@ -1,4 +1,6 @@
-using CodexContractsPlugin;
+using BlockchainUtils;
+using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;

@@ -18,18 +20,29 @@ namespace GethConnector
                return null;
            }

+            var gethNode = new CustomGethNode(log, new BlockCache(), GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+
+            var config = GetCodexMarketplaceConfig(gethNode, GethInput.MarketplaceAddress);
+
            var contractsDeployment = new CodexContractsDeployment(
+                config: config,
                marketplaceAddress: GethInput.MarketplaceAddress,
                abi: GethInput.ABI,
                tokenAddress: GethInput.TokenAddress
            );

-            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
            var contracts = new CodexContractsAccess(log, gethNode, contractsDeployment);

            return new GethConnector(gethNode, contracts);
        }

+        private static MarketplaceConfig GetCodexMarketplaceConfig(IGethNode gethNode, string marketplaceAddress)
+        {
+            var func = new ConfigurationFunctionBase();
+            var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
+            return response.ReturnValue1;
+        }
+
        private GethConnector(IGethNode gethNode, ICodexContracts codexContracts)
        {
            GethNode = gethNode;
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
  </PropertyGroup>
@@ -45,8 +45,12 @@
        private static string? GetEnvVar(List<string> error, string name)
        {
            var result = Environment.GetEnvironmentVariable(name);
-            if (string.IsNullOrEmpty(result)) error.Add($"'{name}' is not set.");
-            return result;
+            if (string.IsNullOrEmpty(result))
+            {
+                error.Add($"'{name}' is not set.");
+                return null;
+            }
+            return result.Trim();
        }
    }
 }
@@ -1,41 +0,0 @@
-using Utils;
-
-namespace KubernetesWorkflow
-{
-    public static class ByteSizeExtensions
-    {
-        public static string ToSuffixNotation(this ByteSize b)
-        {
-            long x = 1024;
-            var map = new Dictionary<long, string>
-            {
-                { Pow(x, 4), "Ti" },
-                { Pow(x, 3), "Gi" },
-                { Pow(x, 2), "Mi" },
-                { (x), "Ki" },
-            };
-
-            var bytes = b.SizeInBytes;
-            foreach (var pair in map)
-            {
-                if (bytes > pair.Key)
-                {
-                    double bytesD = bytes;
-                    double divD = pair.Key;
-                    double numD = Math.Ceiling(bytesD / divD);
-                    var v = Convert.ToInt64(numD);
-                    return $"{v}{pair.Value}";
-                }
-            }
-
-            return $"{bytes}";
-        }
-
-        private static long Pow(long x, int v)
-        {
-            long result = 1;
-            for (var i = 0; i < v; i++) result *= x;
-            return result;
-        }
-    }
-}
@@ -3,7 +3,7 @@ using Logging;

 namespace KubernetesWorkflow
 {
-    public class CrashWatcher
+    public class ContainerCrashWatcher
    {
        private readonly ILog log;
        private readonly KubernetesClientConfiguration config;
@@ -11,12 +11,11 @@ namespace KubernetesWorkflow
        private readonly string podName;
        private readonly string recipeName;
        private readonly string k8sNamespace;
-        private ILogHandler? logHandler;
        private CancellationTokenSource cts;
        private Task? worker;
        private Exception? workerException;

-        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
+        public ContainerCrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
        {
            this.log = log;
            this.config = config;
@@ -27,11 +26,10 @@ namespace KubernetesWorkflow
            cts = new CancellationTokenSource();
        }

-        public void Start(ILogHandler logHandler)
+        public void Start()
        {
            if (worker != null) throw new InvalidOperationException();

-            this.logHandler = logHandler;
            cts = new CancellationTokenSource();
            worker = Task.Run(Worker);
        }
@@ -47,10 +45,12 @@ namespace KubernetesWorkflow
            if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
        }

-        public bool HasContainerCrashed()
+        public bool HasCrashed()
        {
            using var client = new Kubernetes(config);
-            return HasContainerBeenRestarted(client);
+            var result = HasContainerBeenRestarted(client);
+            if (result) DownloadCrashedContainerLogs(client);
+            return result;
        }

        private void Worker()
@@ -70,27 +70,29 @@ namespace KubernetesWorkflow
            using var client = new Kubernetes(config);
            while (!token.IsCancellationRequested)
            {
-                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10));
-
                if (HasContainerBeenRestarted(client))
                {
                    DownloadCrashedContainerLogs(client);
                    return;
                }
+
+                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10));
            }
        }

        private bool HasContainerBeenRestarted(Kubernetes client)
        {
            var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
-            return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            if (result) log.Log("Pod crash detected for " + containerName);
+            return result;
        }

        private void DownloadCrashedContainerLogs(Kubernetes client)
        {
-            log.Log("Pod crash detected for " + containerName);
            using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
-            logHandler!.Log(stream);
+            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName, containerName);
+            handler.Log(stream);
        }
    }
 }
@@ -16,6 +16,7 @@ namespace KubernetesWorkflow
        {
            var config = GetConfig();
            UpdateHostAddress(config);
+            config.SkipTlsVerify = true; // Required for operation on Wings cluster.
            return config;
        }

@@ -45,7 +45,7 @@ namespace KubernetesWorkflow

        public void WaitUntilOnline(RunningContainer container)
        {
-            WaitUntilDeploymentOnline(container.Recipe.Name);
+            WaitUntilDeploymentOnline(container);
        }

        public PodInfo GetPodInfo(RunningDeployment deployment)
@@ -64,14 +64,14 @@ namespace KubernetesWorkflow
            if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
        }

-        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
+        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous)
        {
            log.Debug();

            var podName = GetPodName(container);
            var recipeName = container.Recipe.Name;

-            using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines, previous: previous));
            logHandler.Log(stream);
        }

@@ -115,34 +115,47 @@ namespace KubernetesWorkflow
            });
        }

-        public void DeleteAllNamespacesStartingWith(string prefix)
+        public void DeleteAllNamespacesStartingWith(string prefix, bool wait)
        {
            log.Debug();

            var all = client.Run(c => c.ListNamespace().Items);
            var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(prefix));

+            if (wait)
+            {
+                // If we're going to wait, trigger the delete for all the namespaces immediately.
+                // Then wait for them to finish one by one.
+                foreach (var ns in namespaces)
+                {
+                    DeleteNamespace(ns, false);
+                }
+            }
+
            foreach (var ns in namespaces)
            {
-                DeleteNamespace(ns);
+                DeleteNamespace(ns, wait);
            }
        }

-        public void DeleteNamespace()
+        public void DeleteNamespace(bool wait)
        {
            log.Debug();
            if (IsNamespaceOnline(K8sNamespace))
            {
                client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
+
+                if (wait) WaitUntilNamespaceDeleted(K8sNamespace);
            }
        }

-        public void DeleteNamespace(string ns)
+        public void DeleteNamespace(string ns, bool wait)
        {
            log.Debug();
            if (IsNamespaceOnline(ns))
            {
                client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
+                if (wait) WaitUntilNamespaceDeleted(ns);
            }
        }

@@ -532,7 +545,7 @@ namespace KubernetesWorkflow
            }
            if (set.Memory.SizeInBytes != 0)
            {
-                result.Add("memory", new ResourceQuantity(set.Memory.ToSuffixNotation()));
+                result.Add("memory", new ResourceQuantity(set.Memory.SizeInBytes.ToString()));
            }
            return result;
        }
@@ -732,7 +745,10 @@ namespace KubernetesWorkflow
                throw new Exception($"Expected to find 1 pod by podLabel '{deployment.PodLabel}'. Found: {pods.Length}. " +
                    $"Total number of pods: {allPods.Items.Count}. Their labels: {string.Join(Environment.NewLine, allLabels)}");
            }
-            return pods[0];
+            var pod = pods[0];
+            if (pod.Status == null) throw new Exception("Pod status unknown");
+            if (string.IsNullOrEmpty(pod.Status.PodIP)) throw new Exception("Pod IP unknown");
+            return pod;
        }

        #endregion
@@ -871,15 +887,44 @@ namespace KubernetesWorkflow
            WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated));
        }

-        private void WaitUntilDeploymentOnline(string deploymentName)
+        private void WaitUntilNamespaceDeleted(string @namespace)
+        {
+            WaitUntil(() => !IsNamespaceOnline(@namespace), nameof(WaitUntilNamespaceDeleted));
+        }
+
+        private void WaitUntilDeploymentOnline(RunningContainer container)
        {
            WaitUntil(() =>
            {
-                var deployment = client.Run(c => c.ReadNamespacedDeployment(deploymentName, K8sNamespace));
+                CheckForCrash(container);
+
+                var deployment = client.Run(c => c.ReadNamespacedDeployment(container.Recipe.Name, K8sNamespace));
                return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
            }, nameof(WaitUntilDeploymentOnline));
        }

+        private void CheckForCrash(RunningContainer container)
+        {
+            var deploymentName = container.Recipe.Name;
+            var podName = GetPodName(container);
+
+            var podInfo = client.Run(c => c.ReadNamespacedPod(podName, K8sNamespace));
+            if (podInfo == null) return;
+            if (podInfo.Status == null) return;
+            if (podInfo.Status.ContainerStatuses == null) return;
+
+            var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            if (result)
+            {
+                var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
+                log.Error(msg);
+
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg, deploymentName), tailLines: null, previous: true);
+
+                throw new Exception(msg);
+            }
+        }
+
        private void WaitUntilDeploymentOffline(string deploymentName)
        {
            WaitUntil(() =>
@@ -914,13 +959,13 @@ namespace KubernetesWorkflow

        #endregion

-        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        public ContainerCrashWatcher CreateCrashWatcher(RunningContainer container)
        {
            var containerName = container.Name;
            var podName = GetPodName(container);
            var recipeName = container.Recipe.Name;

-            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
+            return new ContainerCrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
        }

        private V1Pod[] FindPodsByLabel(string podLabel)
@@ -24,8 +24,8 @@
                .Replace("]", "-")
                .Replace(",", "-");

-            result = result.Trim('-');
            if (result.Length > maxLength) result = result.Substring(0, maxLength);
+            result = result.Trim('-');

            return result;
        }
@@ -0,0 +1,42 @@
+namespace Core
+{
+    public interface IK8sTimeSet
+    {
+        /// <summary>
+        /// After a failed K8s operation, wait this long before trying again.
+        /// </summary>
+        TimeSpan K8sOperationRetryDelay();
+
+        /// <summary>
+        /// Maximum total time to attempt to perform a successful k8s operation.
+        /// If k8s operations fail during this timespan, retries will be made.
+        /// </summary>
+        TimeSpan K8sOperationTimeout();
+    }
+
+    public class DefaultK8sTimeSet : IK8sTimeSet
+    {
+        public TimeSpan K8sOperationRetryDelay()
+        {
+            return TimeSpan.FromSeconds(10);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromMinutes(30);
+        }
+    }
+
+    public class LongK8sTimeSet : IK8sTimeSet
+    {
+        public TimeSpan K8sOperationRetryDelay()
+        {
+            return TimeSpan.FromSeconds(30);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromHours(1);
+        }
+    }
+}
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>KubernetesWorkflow</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>
@@ -1,4 +1,6 @@
-namespace KubernetesWorkflow
+using Logging;
+
+namespace KubernetesWorkflow
 {
    public interface ILogHandler
    {
@@ -20,4 +22,25 @@

        protected abstract void ProcessLine(string line);
    }
+
+    public class WriteToFileLogHandler : LogHandler, ILogHandler
+    {
+        public WriteToFileLogHandler(ILog sourceLog, string description, string addFileName)
+        {
+            LogFile = sourceLog.CreateSubfile(addFileName);
+
+            var msg = $"{description} -->> {LogFile.Filename}";
+            sourceLog.Log(msg);
+
+            LogFile.Write(msg);
+            LogFile.WriteRaw(description);
+        }
+
+        public LogFile LogFile { get; }
+
+        protected override void ProcessLine(string line)
+        {
+            LogFile.WriteRaw(line);
+        }
+    }
 }
@@ -1,4 +1,5 @@
-using Newtonsoft.Json.Linq;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;

 namespace KubernetesWorkflow.Recipe
 {
@@ -21,14 +22,13 @@ namespace KubernetesWorkflow.Recipe
            var typeName = GetTypeName(typeof(T));
            var userData = Additionals.SingleOrDefault(a => a.Type == typeName);
            if (userData == null) return default;
-            var jobject = (JObject)userData.UserData;
-            return jobject.ToObject<T>();
+            return JsonConvert.DeserializeObject<T>(userData.UserData);
        }

        private static Additional ConvertToAdditional(object userData)
        {
            var typeName = GetTypeName(userData.GetType());
-            return new Additional(typeName, userData);
+            return new Additional(typeName, JsonConvert.SerializeObject(userData));
        }

        private static string GetTypeName(Type type)
@@ -41,13 +41,13 @@ namespace KubernetesWorkflow.Recipe

    public class Additional
    {
-        public Additional(string type, object userData)
+        public Additional(string type, string userData)
        {
            Type = type;
            UserData = userData;
        }

        public string Type { get; }
-        public object UserData { get; }
+        public string UserData { get; }
    }
 }
@@ -2,8 +2,9 @@
 {
    public class ContainerRecipe
    {
-        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, CommandOverride commandOverride, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
+        public ContainerRecipe(DateTime recipeCreatedUtc, int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, CommandOverride commandOverride, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
        {
+            RecipeCreatedUtc = recipeCreatedUtc;
            Number = number;
            NameOverride = nameOverride;
            Image = image;
@@ -31,6 +32,7 @@
            if (exposedPorts.Any(p => string.IsNullOrEmpty(p.Tag))) throw new Exception("Port tags are required for all exposed ports.");
        }

+        public DateTime RecipeCreatedUtc { get; }
        public string Name { get; }
        public int Number { get; }
        public string? NameOverride { get; }
@@ -25,7 +25,7 @@ namespace KubernetesWorkflow.Recipe

            Initialize(config);

-            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, commandOverride, setCriticalPriority,
+            var recipe = new ContainerRecipe(DateTime.UtcNow, containerNumber, config.NameOverride, Image, resources, schedulingAffinity, commandOverride, setCriticalPriority,
                exposedPorts.ToArray(),
                internalPorts.ToArray(),
                envVars.ToArray(),
@@ -105,7 +105,7 @@ namespace KubernetesWorkflow.Recipe

        protected void AddVolume(string name, string mountPath, string? subPath = null, string? secret = null, string? hostPath = null)
        {
-            var size = 10.MB().ToSuffixNotation();
+            var size = 10.MB().SizeInBytes.ToString();
            volumeMounts.Add(new VolumeMount(name, mountPath, subPath, size, secret, hostPath));
        }

@@ -114,7 +114,7 @@ namespace KubernetesWorkflow.Recipe
            volumeMounts.Add(new VolumeMount(
                $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}",
                mountPath,
-                resourceQuantity: volumeSize.ToSuffixNotation()));
+                resourceQuantity: volumeSize.SizeInBytes.ToString()));
        }

        protected void Additional(object userData)
@@ -1,8 +1,5 @@
-using k8s;
-using k8s.Models;
-using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Recipe;
 using KubernetesWorkflow.Types;
-using Newtonsoft.Json;

 namespace KubernetesWorkflow
 {
@@ -13,12 +13,13 @@ namespace KubernetesWorkflow
        FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
        PodInfo GetPodInfo(RunningContainer container);
        PodInfo GetPodInfo(RunningPod pod);
-        CrashWatcher CreateCrashWatcher(RunningContainer container);
+        ContainerCrashWatcher CreateCrashWatcher(RunningContainer container);
        void Stop(RunningPod pod, bool waitTillStopped);
-        void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
+        void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
+        IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null);
        string ExecuteCommand(RunningContainer container, string command, params string[] args);
-        void DeleteNamespace();
-        void DeleteNamespacesStartingWith(string namespacePrefix);
+        void DeleteNamespace(bool wait);
+        void DeleteNamespacesStartingWith(string namespacePrefix, bool wait);
    }

    public class StartupWorkflow : IStartupWorkflow
@@ -60,7 +61,8 @@ namespace KubernetesWorkflow
                var startResult = controller.BringOnline(recipes, location);
                var containers = CreateContainers(startResult, recipes, startupConfig);

-                var rc = new RunningPod(startupConfig, startResult, containers);
+                var info = GetPodInfo(startResult.Deployment);
+                var rc = new RunningPod(Guid.NewGuid().ToString(), info, startupConfig, startResult, containers);
                cluster.Configuration.Hooks.OnContainersStarted(rc);

                if (startResult.ExternalService != null)
@@ -82,9 +84,14 @@ namespace KubernetesWorkflow
            });
        }

+        public PodInfo GetPodInfo(RunningDeployment deployment)
+        {
+            return K8s(c => c.GetPodInfo(deployment));
+        }
+
        public PodInfo GetPodInfo(RunningContainer container)
        {
-            return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment));
+            return GetPodInfo(container.RunningPod.StartResult.Deployment);
        }

        public PodInfo GetPodInfo(RunningPod pod)
@@ -92,28 +99,50 @@ namespace KubernetesWorkflow
            return K8s(c => c.GetPodInfo(pod.StartResult.Deployment));
        }

-        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        public ContainerCrashWatcher CreateCrashWatcher(RunningContainer container)
        {
            return K8s(c => c.CreateCrashWatcher(container));
        }

        public void Stop(RunningPod runningPod, bool waitTillStopped)
        {
+            if (runningPod.IsStopped) return;
+            foreach (var c in runningPod.Containers)
+            {
+                c.StopLog = DownloadContainerLog(c);
+            }
+            runningPod.IsStopped = true;
+
            K8s(controller =>
            {
                controller.Stop(runningPod.StartResult, waitTillStopped);
-                cluster.Configuration.Hooks.OnContainersStopped(runningPod);
            });
+
+            cluster.Configuration.Hooks.OnContainersStopped(runningPod);
        }

-        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null)
+        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null)
        {
            K8s(controller =>
            {
-                controller.DownloadPodLog(container, logHandler, tailLines);
+                controller.DownloadPodLog(container, logHandler, tailLines, previous);
            });
        }

+        public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null)
+        {
+            var msg = $"Downloading container log for '{container.Name}'";
+            log.Log(msg);
+            var logHandler = new WriteToFileLogHandler(log, msg, container.Name);
+
+            K8s(controller =>
+            {
+                controller.DownloadPodLog(container, logHandler, tailLines, previous);
+            });
+
+            return new DownloadedLog(logHandler.LogFile, container.Name);
+        }
+
        public string ExecuteCommand(RunningContainer container, string command, params string[] args)
        {
            return K8s(controller =>
@@ -122,19 +151,19 @@ namespace KubernetesWorkflow
            });
        }

-        public void DeleteNamespace()
+        public void DeleteNamespace(bool wait)
        {
            K8s(controller =>
            {
-                controller.DeleteNamespace();
+                controller.DeleteNamespace(wait);
            });
        }

-        public void DeleteNamespacesStartingWith(string namespacePrefix)
+        public void DeleteNamespacesStartingWith(string namespacePrefix, bool wait)
        {
            K8s(controller =>
            {
-                controller.DeleteAllNamespacesStartingWith(namespacePrefix);
+                controller.DeleteAllNamespacesStartingWith(namespacePrefix, wait);
            });
        }

@@ -147,7 +176,7 @@ namespace KubernetesWorkflow
                var addresses = CreateContainerAddresses(startResult, r);
                log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}");

-                return new RunningContainer(name, r, addresses);
+                return new RunningContainer(Guid.NewGuid().ToString(), name, r, addresses);

            }).ToArray();
        }
@@ -186,6 +215,7 @@ namespace KubernetesWorkflow
            var port = startResult.GetExternalServicePorts(recipe, tag);

            return new Address(
+                logName: $"{recipe.Name}:{tag}",
                startResult.Cluster.HostAddress,
                port.Number);
        }
@@ -197,6 +227,7 @@ namespace KubernetesWorkflow
            var port = startResult.GetInternalServicePorts(recipe, tag);

            return new Address(
+                logName: $"{serviceName}:{tag}",
                $"http://{serviceName}.{namespaceName}.svc.cluster.local",
                port.Number);
        }
@@ -7,27 +7,29 @@ namespace KubernetesWorkflow.Types
 {
    public class RunningContainer
    {
-        public RunningContainer(string name, ContainerRecipe recipe, ContainerAddress[] addresses)
+        public RunningContainer(string id, string name, ContainerRecipe recipe, ContainerAddress[] addresses)
        {
+            Id = id;
            Name = name;
            Recipe = recipe;
            Addresses = addresses;
        }

+        public string Id { get; }
        public string Name { get; }
        public ContainerRecipe Recipe { get; }
        public ContainerAddress[] Addresses { get; }
+        public IDownloadedLog? StopLog { get; internal set; }

        [JsonIgnore]
        public RunningPod RunningPod { get; internal set; } = null!;

-        public Address GetAddress(ILog log, string portTag)
+        public Address GetAddress(string portTag)
        {
            var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
            if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);

            var select = SelectAddress(addresses);
-            log.Debug($"Container '{Name}' selected for tag '{portTag}' address: '{select}'");
            return select.Address;
        }

@@ -50,5 +52,21 @@ namespace KubernetesWorkflow.Types
            }
            throw new Exception("Running location not known.");
        }
+
+        public override string ToString()
+        {
+            return Name;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is RunningContainer container &&
+                   Id == container.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
    }
 }
@@ -4,8 +4,10 @@ namespace KubernetesWorkflow.Types
 {
    public class RunningPod
    {
-        public RunningPod(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        public RunningPod(string id, PodInfo podInfo, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
        {
+            Id = id;
+            PodInfo = podInfo;
            StartupConfig = startupConfig;
            StartResult = startResult;
            Containers = containers;
@@ -13,6 +15,8 @@ namespace KubernetesWorkflow.Types
            foreach (var c in containers) c.RunningPod = this;
        }

+        public string Id { get; }
+        public PodInfo PodInfo { get; }
        public StartupConfig StartupConfig { get; }
        public StartResult StartResult { get; }
        public RunningContainer[] Containers { get; }
@@ -23,10 +27,30 @@ namespace KubernetesWorkflow.Types
            get { return $"'{string.Join("&", Containers.Select(c => c.Name).ToArray())}'"; }
        }

+        [JsonIgnore]
+        public bool IsStopped { get; internal set; }
+
        public string Describe()
        {
            return string.Join(",", Containers.Select(c => c.Name));
        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is RunningPod pod &&
+                   Id == pod.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
+
+        public override string ToString()
+        {
+            if (IsStopped) return Name + " (*)";
+            return Name;
+        }
    }

    public static class RunningContainersExtensions
@@ -8,7 +8,7 @@ namespace Logging
        void Debug(string message = "", int skipFrames = 0);
        void Error(string message);
        void AddStringReplace(string from, string to);
-        LogFile CreateSubfile(string ext = "log");
+        LogFile CreateSubfile(string addName, string ext = "log");
    }

    public abstract class BaseLog : ILog
@@ -34,7 +34,7 @@ namespace Logging
        {
            get
            {
-                if (logFile == null) logFile = new LogFile(GetFullName(), "log");
+                if (logFile == null) logFile = new LogFile(GetFullName() + ".log");
                return logFile;
            }
        }
@@ -69,15 +69,19 @@ namespace Logging

        public virtual void Delete()
        {
-            File.Delete(LogFile.FullFilename);
+            File.Delete(LogFile.Filename);
        }

-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
        {
-            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
+            addName = addName
+                .Replace("<", "")
+                .Replace(">", "");
+
+            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}_{addName}.{ext}");
        }

-        private string ApplyReplacements(string str)
+        protected string ApplyReplacements(string str)
        {
            if (IsDebug) return str;
            foreach (var replacement in replacements)
@@ -9,7 +9,7 @@

        public override void Log(string message)
        {
-            Console.WriteLine(message);
+            Console.WriteLine(ApplyReplacements(message));
        }
    }
 }
@@ -0,0 +1,79 @@
+namespace Logging
+{
+    public interface IDownloadedLog
+    {
+        string SourceName { get; }
+
+        void IterateLines(Action<string> action);
+        void IterateLines(Action<string> action, params string[] thatContain);
+        string[] GetLinesContaining(string expectedString);
+        string[] FindLinesThatContain(params string[] tags);
+        string GetFilepath();
+        void DeleteFile();
+    }
+
+    public class DownloadedLog : IDownloadedLog
+    {
+        private readonly LogFile logFile;
+
+        public DownloadedLog(string filepath, string sourceName)
+        {
+            logFile = new LogFile(filepath);
+            SourceName = sourceName;
+        }
+
+        public DownloadedLog(LogFile logFile, string sourceName)
+        {
+            this.logFile = logFile;
+            SourceName = sourceName;
+        }
+
+        public string SourceName { get; }
+
+        public void IterateLines(Action<string> action)
+        {
+            using var file = File.OpenRead(logFile.Filename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                action(line);
+                line = streamReader.ReadLine();
+            }
+        }
+
+        public void IterateLines(Action<string> action, params string[] thatContain)
+        {
+            IterateLines(line =>
+            {
+                if (thatContain.All(line.Contains))
+                {
+                    action(line);
+                }
+            });
+        }
+
+        public string[] GetLinesContaining(string expectedString)
+        {
+            return FindLinesThatContain([expectedString]);
+        }
+
+        public string[] FindLinesThatContain(params string[] tags)
+        {
+            var result = new List<string>();
+            IterateLines(result.Add, tags);
+            return result.ToArray();
+        }
+
+        public string GetFilepath()
+        {
+            return logFile.Filename;
+        }
+
+        public void DeleteFile()
+        {
+            File.Delete(logFile.Filename);
+        }
+    }
+}
@@ -1,21 +1,19 @@
-namespace Logging
+using Utils;
+
+namespace Logging
 {
    public class LogFile
    {
-        private readonly string extension;
        private readonly object fileLock = new object();
-        private string filename;

-        public LogFile(string filename, string extension)
+        public LogFile(string filename)
        {
-            this.filename = filename;
-            this.extension = extension;
-            FullFilename = filename + "." + extension;
+            Filename = filename;

            EnsurePathExists(filename);
        }

-        public string FullFilename { get; private set; }
+        public string Filename { get; private set; }

        public void Write(string message)
        {
@@ -28,7 +26,7 @@
            {
                lock (fileLock)
                { 
-                    File.AppendAllLines(FullFilename, new[] { message });
+                    File.AppendAllLines(Filename, new[] { message });
                }
            }
            catch (Exception ex)
@@ -37,19 +35,24 @@
            }
        }

-        public void ConcatToFilename(string toAdd)
+        public void WriteRawMany(IEnumerable<string> lines)
        {
-            var oldFullName = FullFilename;
-
-            filename += toAdd;
-            FullFilename = filename + "." + extension;
-
-            File.Move(oldFullName, FullFilename);
+            try
+            {
+                lock (fileLock)
+                {
+                    File.AppendAllLines(Filename, lines);
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine("Writing to log has failed: " + ex);
+            }
        }

        private static string GetTimestamp()
        {
-            return $"[{DateTime.UtcNow.ToString("o")}]";
+            return $"[{Time.FormatTimestamp(DateTime.UtcNow)}]";
        }

        private void EnsurePathExists(string filename)
@@ -18,9 +18,9 @@
        public string Prefix { get; set; } = string.Empty;


-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
        {
-            return backingLog.CreateSubfile(ext);
+            return backingLog.CreateSubfile(addName, ext);
        }

        public void Debug(string message = "", int skipFrames = 0)
@@ -14,9 +14,9 @@
            OnAll(l => l.AddStringReplace(from, to));
        }

-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
        {
-            return targetLogs.First().CreateSubfile(ext);
+            return targetLogs.First().CreateSubfile(addName, ext);
        }

        public void Debug(string message = "", int skipFrames = 0)
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Logging</RootNamespace>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
@@ -1,25 +1,25 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
-using NethereumWorkflow.BlockUtils;
 using Utils;

 namespace NethereumWorkflow
 {
    public class NethereumInteraction
    {
-        // BlockCache is a static instance: It stays alive for the duration of the application runtime.
-        private readonly static BlockCache blockCache = new BlockCache();
+        private readonly BlockCache blockCache;

        private readonly ILog log;
        private readonly Web3 web3;

-        internal NethereumInteraction(ILog log, Web3 web3)
+        internal NethereumInteraction(ILog log, Web3 web3, BlockCache blockCache)
        {
            this.log = log;
            this.web3 = web3;
+            this.blockCache = blockCache;
        }

        public string SendEth(string toAddress, decimal ethAmount)
@@ -50,6 +50,21 @@ namespace NethereumWorkflow
            return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function));
        }

+        public TResult Call<TFunction, TResult>(string contractAddress, TFunction function, ulong blockNumber) where TFunction : FunctionMessage, new()
+        {
+            log.Debug(typeof(TFunction).ToString());
+            var handler = web3.Eth.GetContractQueryHandler<TFunction>();
+            return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function, new BlockParameter(blockNumber)));
+        }
+
+        public void Call<TFunction>(string contractAddress, TFunction function, ulong blockNumber) where TFunction : FunctionMessage, new()
+        {
+            log.Debug(typeof(TFunction).ToString());
+            var handler = web3.Eth.GetContractQueryHandler<TFunction>();
+            var result = Time.Wait(handler.QueryRawAsync(contractAddress, function, new BlockParameter(blockNumber)));
+            var aaaa = 0;
+        }
+
        public string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
        {
            log.Debug();
@@ -117,6 +132,7 @@ namespace NethereumWorkflow
            }

            return new BlockInterval(
+                timeRange: timeRange,
                from: fromBlock.Value,
                to: toBlock.Value
            );
@@ -1,4 +1,5 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.Web3;

 namespace NethereumWorkflow
@@ -6,13 +7,15 @@ namespace NethereumWorkflow
    public class NethereumInteractionCreator
    {
        private readonly ILog log;
+        private readonly BlockCache blockCache;
        private readonly string ip;
        private readonly int port;
        private readonly string privateKey;

-        public NethereumInteractionCreator(ILog log, string ip, int port, string privateKey)
+        public NethereumInteractionCreator(ILog log, BlockCache blockCache, string ip, int port, string privateKey)
        {
            this.log = log;
+            this.blockCache = blockCache;
            this.ip = ip;
            this.port = port;
            this.privateKey = privateKey;
@@ -21,7 +24,7 @@ namespace NethereumWorkflow
        public NethereumInteraction CreateWorkflow()
        {
            log.Debug("Starting interaction to " + ip + ":" + port);
-            return new NethereumInteraction(log, CreateWeb3());
+            return new NethereumInteraction(log, CreateWeb3(), blockCache);
        }

        private Web3 CreateWeb3()
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	<RootNamespace>NethereumWorkflow</RootNamespace>
 	<ImplicitUsings>enable</ImplicitUsings>
 	<Nullable>enable</Nullable>
@@ -12,6 +12,7 @@
 	</ItemGroup>

  <ItemGroup>
+    <ProjectReference Include="..\BlockchainUtils\BlockchainUtils.csproj" />
    <ProjectReference Include="..\Logging\Logging.csproj" />
    <ProjectReference Include="..\Utils\Utils.csproj" />
  </ItemGroup>
@@ -1,16 +1,11 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
 using Utils;

 namespace NethereumWorkflow
 {
-    public interface IWeb3Blocks
-    {
-        ulong GetCurrentBlockNumber();
-        DateTime? GetTimestampForBlock(ulong blockNumber);
-    }
-
    public class Web3Wrapper : IWeb3Blocks
    {
        private readonly Web3 web3;
@@ -0,0 +1,61 @@
+namespace OverwatchTranscript
+{
+    public class ActionQueue
+    {
+        // Using ConcurrentQueue<> here would make this process slower.
+        private readonly object queueLock = new object();
+        private readonly AutoResetEvent signal = new AutoResetEvent(false);
+        private List<Action> queue = new List<Action>();
+        private Task queueWorker = null!;
+        private bool stopping = false;
+
+        public void Start()
+        {
+            queueWorker = Task.Run(QueueWorker);
+        }
+
+        public int Count { get; private set; }
+
+        public void StopAndJoin()
+        {
+            stopping = true;
+            queueWorker.Wait();
+            if (queue.Count > 0) throw new Exception("not all acions handled");
+            queueWorker.Dispose();
+        }
+
+        public void Add(Action action)
+        {
+            if (stopping) throw new Exception("queue stopping");
+
+            lock (queueLock)
+            {
+                queue.Add(action);
+                Count = queue.Count;
+            }
+            signal.Set();
+        }
+
+        private void QueueWorker()
+        {
+            while (true)
+            {
+                signal.WaitOne(10);
+
+                List<Action> work = null!;
+                lock (queueLock)
+                {
+                    work = queue;
+                    queue = new List<Action>();
+                    Count = 0;
+                }
+                if (stopping && !work.Any()) return;
+                
+                foreach (var action in work)
+                {
+                    action();
+                }
+            }
+        }
+    }
+}
@@ -0,0 +1,110 @@
+using Logging;
+using System.Collections.Concurrent;
+
+namespace OverwatchTranscript
+{
+    public class BucketSet
+    {
+        private const int numberOfActiveBuckets = 10;
+        private readonly ILog log;
+        private readonly string workingDir;
+        private readonly object _bucketLock = new object();
+        private readonly List<EventBucketWriter> fullBuckets = new List<EventBucketWriter>();
+        private readonly List<EventBucketWriter> activeBuckets = new List<EventBucketWriter>();
+        private readonly ActionQueue queue = new ActionQueue();
+        private int activeBucketIndex = 0;
+        private bool closed = false;
+        private string internalErrors = string.Empty;
+        
+        public BucketSet(ILog log, string workingDir)
+        {
+            this.log = log;
+            this.workingDir = workingDir;
+
+            for (var i = 0; i < numberOfActiveBuckets;i++)
+            {
+                AddNewBucket();
+            }
+
+            queue.Start();
+        }
+
+        public void Add(DateTime utc, object payload)
+        {
+            if (closed) throw new Exception("Buckets already closed!");
+            queue.Add(() => AddInternal(utc, payload));
+            
+            if (queue.Count > 1000)
+            {
+                Thread.Sleep(1);
+            }
+        }
+
+        public IFinalizedBucket[] FinalizeBuckets()
+        {
+            closed = true;
+            queue.StopAndJoin();
+
+            if (IsEmpty()) throw new Exception("No entries have been added.");
+            if (!string.IsNullOrEmpty(internalErrors)) throw new Exception(internalErrors);
+
+            var buckets = fullBuckets.Concat(activeBuckets).ToArray();
+            log.Debug($"Finalizing {buckets.Length} buckets...");
+
+            var finalized = new ConcurrentBag<IFinalizedBucket>();
+            var tasks = Parallel.ForEach(buckets, b => finalized.Add(b.FinalizeBucket()));
+            if (!tasks.IsCompleted) throw new Exception("Failed to finalize buckets: " + tasks);
+
+            return finalized.ToArray();
+        }
+
+        private bool IsEmpty()
+        {
+            return fullBuckets.All(b => b.Count == 0) && activeBuckets.All(b => b.Count == 0);
+        }
+
+        private void AddInternal(DateTime utc, object payload)
+        {
+            try
+            {
+                lock (_bucketLock)
+                {
+                    var current = activeBuckets[activeBucketIndex];
+                    current.Add(utc, payload);
+                    activeBucketIndex = (activeBucketIndex + 1) % numberOfActiveBuckets;
+
+                    if (current.IsFull)
+                    {
+                        log.Debug("Bucket is full. New bucket...");
+                        fullBuckets.Add(current);
+                        activeBuckets.Remove(current);
+                        AddNewBucket();
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                internalErrors += ex.ToString();
+                log.Error(ex.ToString());
+            }
+        }
+
+        private static int bucketSizeIndex = 0;
+        private static int[] bucketSizes = new[]
+        {
+            10000,
+            15000,
+            20000,
+        };
+
+        private void AddNewBucket()
+        {
+            lock (_bucketLock)
+            {
+                var size = bucketSizes[bucketSizeIndex];
+                bucketSizeIndex = (bucketSizeIndex + 1) % bucketSizes.Length;
+                activeBuckets.Add(new EventBucketWriter(log, Path.Combine(workingDir, Guid.NewGuid().ToString()), size));
+            }
+        }
+    }
+}
@@ -0,0 +1,151 @@
+using Logging;
+using Newtonsoft.Json;
+using System.Collections.Concurrent;
+
+namespace OverwatchTranscript
+{
+    public interface IFinalizedBucket
+    {
+        bool IsEmpty { get; }
+        void Update();
+        DateTime? SeeTopUtc();
+        BucketTop? TakeTop();
+    }
+
+    public class BucketTop
+    {
+        public BucketTop(DateTime utc, OverwatchEvent[] events)
+        {
+            Utc = utc;
+            Events = events;
+        }
+
+        public DateTime Utc { get; }
+        public OverwatchEvent[] Events { get; }
+    }
+
+    public class EventBucketReader : IFinalizedBucket
+    {
+        private readonly string bucketFile;
+        private readonly ConcurrentQueue<BucketTop> topQueue = new ConcurrentQueue<BucketTop>();
+        private readonly AutoResetEvent itemDequeued = new AutoResetEvent(false);
+        private readonly AutoResetEvent itemEnqueued = new AutoResetEvent(false);
+        private bool sourceIsEmpty;
+
+        public EventBucketReader(ILog log, string bucketFile)
+        {
+            this.bucketFile = bucketFile;
+            if (!File.Exists(bucketFile)) throw new Exception("Doesn't exist: " + bucketFile);
+
+            log.Debug("Read Bucket open: " + bucketFile);
+
+            Task.Run(ReadBucket);
+        }
+        
+        public bool IsEmpty { get; private set; }
+
+        public void Update()
+        {
+            if (IsEmpty) return;
+            while (topQueue.Count == 0)
+            {
+                UpdateIsEmpty();
+                if (IsEmpty) return;
+
+                itemDequeued.Set();
+                itemEnqueued.WaitOne(200);
+            }
+        }
+
+        public DateTime? SeeTopUtc()
+        {
+            if (IsEmpty) return null;
+            if (topQueue.TryPeek(out BucketTop? top))
+            {
+                return top.Utc;
+            }
+            return null;
+        }
+
+        public BucketTop? TakeTop()
+        {
+            if (IsEmpty) return null;
+            if (topQueue.TryDequeue(out BucketTop? top))
+            {
+                itemDequeued.Set();
+                return top;
+            }
+            return null;
+        }
+
+        private void ReadBucket()
+        {
+            using var file = File.OpenRead(bucketFile);
+            using var reader = new StreamReader(file);
+
+            while (true)
+            {
+                while (topQueue.Count < 5)
+                {
+                    var top = CreateNewTop(reader);
+                    if (top != null)
+                    {
+                        topQueue.Enqueue(top);
+                        itemEnqueued.Set();
+                    }
+                    else
+                    {
+                        sourceIsEmpty = true;
+                        UpdateIsEmpty();
+                        return;
+                    }
+                }
+
+                itemDequeued.Reset();
+                itemDequeued.WaitOne(5000);
+            }
+        }
+
+        private void UpdateIsEmpty()
+        {
+            var allEmpty = sourceIsEmpty && topQueue.IsEmpty;
+            if (!IsEmpty && allEmpty)
+            {
+                File.Delete(bucketFile);
+                IsEmpty = true;
+            }
+        }
+
+        private EventBucketEntry? nextEntry = null;
+        private BucketTop? CreateNewTop(StreamReader reader)
+        {
+            if (nextEntry == null)
+            {
+                nextEntry = ReadEntry(reader);
+                if (nextEntry == null) return null;
+            }
+
+            var topEntry = nextEntry;
+            var entries = new List<EventBucketEntry>
+            {
+                topEntry
+            };
+
+            nextEntry = ReadEntry(reader);
+            while (nextEntry != null && nextEntry.Utc == topEntry.Utc)
+            {
+                entries.Add(nextEntry);
+                nextEntry = ReadEntry(reader);
+            }
+
+            return new BucketTop(topEntry.Utc, entries.Select(e => e.Event).ToArray());
+        }
+
+        private EventBucketEntry? ReadEntry(StreamReader reader)
+        {
+            var line = reader.ReadLine();
+            if (string.IsNullOrEmpty(line)) return null;
+            return JsonConvert.DeserializeObject<EventBucketEntry>(line);
+        }
+    }
+}
@@ -0,0 +1,113 @@
+using Logging;
+using Newtonsoft.Json;
+
+namespace OverwatchTranscript
+{
+    public class EventBucketWriter
+    {
+        private const int MaxBuffer = 1000;
+
+        private readonly object _lock = new object();
+        private bool closed = false;
+        private readonly ILog log;
+        private readonly string bucketFile;
+        private readonly int maxCount;
+        private readonly List<EventBucketEntry> buffer = new List<EventBucketEntry>();
+
+        public EventBucketWriter(ILog log, string bucketFile, int maxCount)
+        {
+            this.log = log;
+            this.bucketFile = bucketFile;
+            this.maxCount = maxCount;
+            if (File.Exists(bucketFile)) throw new Exception("Already exists");
+
+            log.Debug("Write Bucket open: " + bucketFile);
+        }
+
+        public int Count { get; private set; }
+        public bool IsFull { get; private set; }
+
+        public void Add(DateTime utc, object payload)
+        {
+            lock (_lock)
+            {
+                if (closed) throw new Exception("Already closed");
+                AddToBuffer(utc, payload);
+                BufferToFile(emptyBuffer: false);
+            }
+        }
+
+        public IFinalizedBucket FinalizeBucket()
+        {
+            lock (_lock)
+            {
+                closed = true;
+                BufferToFile(emptyBuffer: true);
+                SortFileByTimestamps();
+            }
+            log.Debug($"Finalized bucket with {Count} entries");
+            return new EventBucketReader(log, bucketFile);
+        }
+
+        public override string ToString()
+        {
+            return $"EventBucket: " + Count;
+        }
+
+        private void AddToBuffer(DateTime utc, object payload)
+        {  
+            var typeName = payload.GetType().FullName;
+            if (string.IsNullOrEmpty(typeName)) throw new Exception("Empty typename for payload");
+            if (utc == default) throw new Exception("DateTimeUtc not set");
+
+            var entry = new EventBucketEntry
+            {
+                Utc = utc,
+                Event = new OverwatchEvent
+                {
+                    Type = typeName,
+                    Payload = Json.Serialize(payload)
+                }
+            };
+
+            Count++;
+            IsFull = Count > maxCount;
+
+            buffer.Add(entry);
+        }
+
+        private void BufferToFile(bool emptyBuffer)
+        {
+            if (emptyBuffer || buffer.Count > MaxBuffer)
+            {
+                using var file = File.Open(bucketFile, FileMode.Append);
+                using var writer = new StreamWriter(file);
+                foreach (var entry in buffer)
+                {
+                    writer.WriteLine(Json.Serialize(entry));
+                }
+                log.Debug($"Bucket wrote {buffer.Count} entries to file.");
+                buffer.Clear();
+            }
+        }
+
+        private void SortFileByTimestamps()
+        {
+            var lines = File.ReadAllLines(bucketFile);
+            var entries = lines.Select(Json.Deserialize<EventBucketEntry>)
+                .Cast<EventBucketEntry>()
+                .OrderBy(e => e.Utc)
+                .ToArray();
+
+            File.Delete(bucketFile);
+            File.WriteAllLines(bucketFile, entries.Select(e => Json.Serialize(e)));
+        }
+    }
+
+    [Serializable]
+    public class EventBucketEntry
+    {
+        public DateTime Utc { get; set; }
+        public OverwatchEvent Event { get; set; } = new();
+    }
+}
@@ -0,0 +1,27 @@
+using Newtonsoft.Json;
+using System.Globalization;
+
+namespace OverwatchTranscript
+{
+    public static class Json
+    {
+        private static JsonSerializerSettings settings = new JsonSerializerSettings
+        {
+            Formatting = Formatting.None,
+            NullValueHandling = NullValueHandling.Ignore,
+            Culture = CultureInfo.InvariantCulture,
+            DateFormatHandling = DateFormatHandling.IsoDateFormat,
+            FloatFormatHandling = FloatFormatHandling.Symbol
+        };
+
+        public static string Serialize(object obj, Formatting formatting = Formatting.None)
+        {
+            return JsonConvert.SerializeObject(obj, formatting, settings);
+        }
+
+        public static T Deserialize<T>(string json)
+        {
+            return JsonConvert.DeserializeObject<T>(json)!;
+        }
+    }
+}
@@ -0,0 +1,56 @@
+namespace OverwatchTranscript
+{
+    [Serializable]
+    public class OverwatchTranscript
+    {
+        public OverwatchHeader Header { get; set; } = new();
+        public OverwatchMomentReference[] MomentReferences { get; set; } = Array.Empty<OverwatchMomentReference>();
+    }
+
+    [Serializable]
+    public class OverwatchMomentReference
+    {
+        public string MomentsFile { get; set; } = string.Empty;
+        public int NumberOfMoments { get; set; }
+        public int NumberOfEvents { get; set; }
+        public DateTime EarliestUtc { get; set; }
+        public DateTime LatestUtc { get; set; }
+    }
+
+    [Serializable]
+    public class OverwatchHeader
+    {
+        public OverwatchCommonHeader Common { get; set; } = new();
+        public OverwatchHeaderEntry[] Entries { get; set; } = Array.Empty<OverwatchHeaderEntry>();
+    }
+
+    [Serializable]
+    public class OverwatchCommonHeader
+    {
+        public long NumberOfMoments { get; set; }
+        public long NumberOfEvents { get; set; }
+        public DateTime EarliestUtc { get; set; }
+        public DateTime LatestUtc { get; set; }
+    }
+
+    [Serializable]
+    public class OverwatchHeaderEntry
+    {
+        public string Key { get; set; } = string.Empty;
+        public string Value { get; set; } = string.Empty;
+    }
+
+    [Serializable]
+    public class OverwatchMoment
+    {
+        public DateTime Utc { get; set; }
+        public OverwatchEvent[] Events { get; set; } = Array.Empty<OverwatchEvent>();
+    }
+
+    [Serializable]
+    public class OverwatchEvent
+    {
+        public string Type { get; set; } = string.Empty;
+        public string Payload { get; set; } = string.Empty;
+    }
+}
@@ -0,0 +1,104 @@
+using Newtonsoft.Json;
+
+namespace OverwatchTranscript
+{
+    public class MomentReader
+    {
+        private readonly OverwatchTranscript model;
+        private readonly string workingDir;
+        private int referenceIndex = 0;
+        private int momentsRead = 0;
+        private OpenReference currentRef;
+
+        public MomentReader(OverwatchTranscript model, string workingDir)
+        {
+            this.model = model;
+            this.workingDir = workingDir;
+
+            currentRef = CreateOpenReference();
+        }
+
+        public OverwatchMoment? Next()
+        {
+            if (referenceIndex >= model.MomentReferences.Length) return null;
+
+            var moment = currentRef.ReadNext();
+            if (moment == null)
+            {
+                Close();
+
+                // This reference file ran out.
+                // The number of moments read should match exactly the number of moments
+                // describe in the reference. If not, error:
+                var expected = model.MomentReferences[referenceIndex].NumberOfMoments;
+                if (momentsRead != expected)
+                {
+                    throw new Exception("Number of moments read from referenced file does not match number of moments value in model. " +
+                        $"Reads: { momentsRead} - model.MomentReferences[{referenceIndex}].NumberOfMoment: {expected}"); 
+                }
+
+                referenceIndex++;
+                if (referenceIndex < model.MomentReferences.Length)
+                {
+                    // Proceed to next reference file.
+                    currentRef = CreateOpenReference();
+                    momentsRead = 0;
+                    return Next();
+                }
+                else
+                {
+                    // That was the last one.
+                    return null;
+                }
+            }
+            else
+            {
+                momentsRead++;
+                return moment;
+            }
+        }
+
+        public void Close()
+        {
+            if (currentRef != null)
+            {
+                currentRef.Close();
+                currentRef = null!;
+            }
+        }
+
+        private OpenReference CreateOpenReference()
+        {
+            var filepath = Path.Combine(workingDir, model.MomentReferences[referenceIndex].MomentsFile);
+            return new OpenReference(filepath);
+        }
+
+        private class OpenReference
+        {
+            private readonly FileStream file;
+            private readonly StreamReader reader;
+
+            public OpenReference(string filePath)
+            {
+                file = File.OpenRead(filePath);
+                reader = new StreamReader(file);
+            }
+
+            public OverwatchMoment? ReadNext()
+            {
+                var line = reader.ReadLine();
+                if (string.IsNullOrEmpty(line)) return null;
+                return JsonConvert.DeserializeObject<OverwatchMoment>(line);
+            }
+
+            public void Close()
+            {
+                reader.Close();
+                file.Close();
+
+                reader.Dispose();
+                file.Dispose();
+            }
+        }
+    }
+}
@@ -0,0 +1,153 @@
+using Logging;
+using Newtonsoft.Json;
+
+namespace OverwatchTranscript
+{
+    public class MomentReferenceBuilder
+    {
+        private const int MaxMomentsPerReference = 10000;
+        private readonly ILog log;
+        private readonly string workingDir;
+
+        public MomentReferenceBuilder(ILog log, string workingDir)
+        {
+            this.log = log;
+            this.workingDir = workingDir;
+        }
+
+        public OverwatchMomentReference[] Build(IFinalizedBucket[] finalizedBuckets)
+        {
+            var result = new List<OverwatchMomentReference>();
+            var currentBuilder = new Builder(log, workingDir);
+
+            var buckets = finalizedBuckets.ToList();
+            log.Debug($"Building references for {buckets.Count} buckets.");
+            while (buckets.Any())
+            {
+                foreach (var b in buckets) b.Update();
+
+                buckets.RemoveAll(b => b.IsEmpty);
+                if (!buckets.Any()) break;
+
+                var earliestUtc = GetEarliestUtc(buckets);
+                if (earliestUtc == null) continue;
+                
+                var tops = CollectAllTopsForUtc(earliestUtc.Value, buckets);
+                var moment = ConvertTopsToMoment(tops);
+                currentBuilder.Add(moment);
+                if (currentBuilder.NumberOfMoments == MaxMomentsPerReference)
+                {
+                    result.Add(currentBuilder.Build());
+                    currentBuilder = new Builder(log, workingDir);
+                }
+            }
+
+            if (currentBuilder.NumberOfMoments > 0)
+            {
+                result.Add(currentBuilder.Build());
+            }
+
+            return result.ToArray();
+        }
+
+        private OverwatchMoment ConvertTopsToMoment(List<BucketTop> tops)
+        {
+            var discintUtc = tops.Select(e => e.Utc).Distinct().ToArray();
+            if (discintUtc.Length != 1) throw new Exception("UTC mixing in moment construction.");
+
+            return new OverwatchMoment
+            {
+                Utc = tops[0].Utc,
+                Events = tops.SelectMany(e => e.Events).ToArray()
+            };
+        }
+
+        private List<BucketTop> CollectAllTopsForUtc(DateTime earliestUtc, List<IFinalizedBucket> buckets)
+        {
+            var result = new List<BucketTop>();
+
+            foreach (var bucket in buckets)
+            {
+                if (bucket.IsEmpty) continue;
+
+                var utc = bucket.SeeTopUtc();
+                if (utc == null) continue;
+
+                if (utc.Value == earliestUtc)
+                {
+                    var top = bucket.TakeTop();
+                    if (top == null) throw new Exception("top was null after top utc was not");
+                    result.Add(top);
+                }
+            }
+
+            return result;
+        }
+
+        private DateTime? GetEarliestUtc(List<IFinalizedBucket> buckets)
+        {
+            var earliest = DateTime.MaxValue;
+            foreach (var bucket in buckets)
+            {
+                var utc = bucket.SeeTopUtc();
+                if (utc == null) return null;
+
+                if (utc.Value < earliest) earliest = utc.Value;
+            }
+            return earliest;
+        }
+
+        public class Builder
+        {
+            private readonly ILog log;
+            private readonly string workingDir;
+            private OverwatchMomentReference reference;
+            private readonly ActionQueue queue = new ActionQueue();
+
+            public Builder(ILog log, string workingDir)
+            {
+                reference = new OverwatchMomentReference
+                {
+                    MomentsFile = Guid.NewGuid().ToString(),
+                    EarliestUtc = DateTime.MaxValue,
+                    LatestUtc = DateTime.MinValue,
+                    NumberOfEvents = 0,
+                    NumberOfMoments = 0,
+                };
+                this.log = log;
+                this.workingDir = workingDir;
+                queue.Start();
+            }
+
+            public int NumberOfMoments => reference.NumberOfMoments;
+
+            public void Add(OverwatchMoment moment)
+            {
+                if (moment.Utc < reference.EarliestUtc) reference.EarliestUtc = moment.Utc;
+                if (moment.Utc > reference.LatestUtc) reference.LatestUtc = moment.Utc;
+                reference.NumberOfMoments++;
+                reference.NumberOfEvents += moment.Events.Length;
+
+                var filePath = Path.Combine(workingDir, reference.MomentsFile);
+
+                queue.Add(() =>
+                {
+                    File.AppendAllLines(filePath, new[]
+                    {
+                        Json.Serialize(moment)
+                    });
+                });
+            }
+
+            public OverwatchMomentReference Build()
+            {
+                queue.StopAndJoin();
+
+                log.Debug($"Created reference with {reference.NumberOfMoments} moments and {reference.NumberOfEvents} events...");
+                var result = reference;
+                reference = null!;
+                return result;
+            }
+        }
+    }
+}
@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,23 @@
+using Logging;
+
+namespace OverwatchTranscript
+{
+    public static class Transcript
+    {
+        public static ITranscriptWriter NewWriter(ILog log)
+        {
+            log = new LogPrefixer(log, "(TranscriptWriter) ");
+            return new TranscriptWriter(log, NewWorkDir());
+        }
+
+        public static ITranscriptReader NewReader(string transcriptFile)
+        {
+            return new TranscriptReader(NewWorkDir(), transcriptFile);
+        }
+
+        private static string NewWorkDir()
+        {
+            return Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+        }
+    }
+}
@@ -0,0 +1,8 @@
+namespace OverwatchTranscript
+{
+    public static class TranscriptConstants
+    {
+        public const string TranscriptFilename = "transcript.json";
+        public const string ArtifactFolderName = "artifacts";
+    }
+}
@@ -0,0 +1,264 @@
+using Newtonsoft.Json;
+using System.IO;
+using System;
+using System.IO.Compression;
+using System.Linq;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+
+namespace OverwatchTranscript
+{
+    public interface ITranscriptReader
+    {
+        OverwatchCommonHeader Header { get; }
+        T GetHeader<T>(string key);
+        void AddMomentHandler(Action<ActivateMoment> handler);
+        void AddEventHandler<T>(Action<ActivateEvent<T>> handler);
+        bool Next();
+        void Close();
+    }
+
+    public class TranscriptReader : ITranscriptReader
+    {
+        private readonly object handlersLock = new object();
+        private readonly string transcriptFile;
+        private readonly string artifactsFolder;
+        private readonly List<Action<ActivateMoment>> momentHandlers = new List<Action<ActivateMoment>>();
+        private readonly Dictionary<string, List<Action<ActivateMoment, string>>> eventHandlers = new Dictionary<string, List<Action<ActivateMoment, string>>>();
+        private readonly string workingDir;
+        private readonly OverwatchTranscript model;
+        private bool closed;
+        private long momentCounter;
+        private readonly ConcurrentQueue<OverwatchMoment> queue = new ConcurrentQueue<OverwatchMoment>();
+        private readonly Task queueFiller;
+
+        public TranscriptReader(string workingDir, string inputFilename)
+        {
+            closed = false;
+            this.workingDir = workingDir;
+            transcriptFile = Path.Combine(workingDir, TranscriptConstants.TranscriptFilename);
+            artifactsFolder = Path.Combine(workingDir, TranscriptConstants.ArtifactFolderName);
+
+            if (!Directory.Exists(workingDir)) Directory.CreateDirectory(workingDir);
+            if (File.Exists(transcriptFile) || Directory.Exists(artifactsFolder)) throw new Exception("workingdir not clean");
+
+            model = LoadModel(inputFilename);
+
+            queueFiller = Task.Run(() => FillQueue(model, workingDir));
+        }
+
+        public OverwatchCommonHeader Header
+        {
+            get
+            {
+                CheckClosed();
+                return model.Header.Common;
+            }
+        }
+
+        public T GetHeader<T>(string key)
+        {
+            CheckClosed();
+            var value = model.Header.Entries.First(e => e.Key == key).Value;
+            return JsonConvert.DeserializeObject<T>(value)!;
+        }
+
+        public void AddMomentHandler(Action<ActivateMoment> handler)
+        {
+            CheckClosed();
+            lock (handlersLock)
+            {
+                momentHandlers.Add(handler);
+            }
+        }
+
+        public void AddEventHandler<T>(Action<ActivateEvent<T>> handler)
+        {
+            CheckClosed();
+
+            var typeName = typeof(T).FullName;
+            if (string.IsNullOrEmpty(typeName)) throw new Exception("Empty typename for payload");
+
+            lock (handlersLock)
+            {
+                if (eventHandlers.ContainsKey(typeName))
+                {
+                    eventHandlers[typeName].Add(CreateEventAction(handler));
+                }
+                else
+                {
+                    eventHandlers.Add(typeName, new List<Action<ActivateMoment, string>>
+                    {
+                        CreateEventAction(handler)
+                    });
+                }
+            }
+        }
+
+        private readonly object nextLock = new object();
+        private OverwatchMoment? moment = null;
+        private OverwatchMoment? next = null;
+
+        public bool Next()
+        {
+            CheckClosed();
+
+            OverwatchMoment? m = null;
+            TimeSpan? duration = null;
+            lock (nextLock)
+            {
+                if (next == null)
+                {
+                    if (!queue.TryDequeue(out moment)) return false;
+                    queue.TryDequeue(out next);
+                }
+                else
+                {
+                    moment = next;
+                    next = null;
+                    queue.TryDequeue(out next);
+                }
+
+                m = moment;
+                duration = GetMomentDuration();
+            }
+
+            ActivateMoment(moment, duration);
+
+            return true;
+        }
+
+        public void Close()
+        {
+            CheckClosed();
+            closed = true;
+
+            queueFiller.Wait();
+
+            Directory.Delete(workingDir, true);
+        }
+
+        private Action<ActivateMoment, string> CreateEventAction<T>(Action<ActivateEvent<T>> handler)
+        {
+            return (m, s) =>
+            {
+                handler(new ActivateEvent<T>(m, JsonConvert.DeserializeObject<T>(s)!));
+            };
+        }
+
+        private void FillQueue(OverwatchTranscript model, string workingDir)
+        {
+            var reader = new MomentReader(model, workingDir);
+
+            while (true)
+            {
+                if (closed)
+                {
+                    reader.Close();
+                    return;
+                }
+
+                while (queue.Count < 10)
+                {
+                    var moment = reader.Next();
+                    if (moment == null)
+                    {
+                        reader.Close();
+                        return;
+                    }
+                    queue.Enqueue(moment);
+                }
+
+                Thread.Sleep(1);
+            }
+        }
+
+        private TimeSpan? GetMomentDuration()
+        {
+            if (moment == null) return null;
+            if (next == null) return null;
+
+            return next.Utc - moment.Utc;
+        }
+
+        private void ActivateMoment(OverwatchMoment moment, TimeSpan? duration)
+        {
+            var m = new ActivateMoment(moment.Utc, duration, momentCounter);
+
+            lock (handlersLock)
+            {
+                ActivateMomentHandlers(m);
+
+                foreach (var @event in moment.Events)
+                {
+                    ActivateEventHandlers(m, @event);
+                }
+            }
+
+            momentCounter++;
+        }
+
+        private void ActivateMomentHandlers(ActivateMoment m)
+        {
+            foreach (var handler in momentHandlers)
+            {
+                handler(m);
+            }
+        }
+
+        private void ActivateEventHandlers(ActivateMoment m, OverwatchEvent @event)
+        {
+            if (!eventHandlers.ContainsKey(@event.Type)) return;
+            var handlers = eventHandlers[@event.Type];
+
+            foreach (var handler in handlers)
+            {
+                handler(m, @event.Payload);
+            }
+        }
+
+        private OverwatchTranscript LoadModel(string inputFilename)
+        {
+            ZipFile.ExtractToDirectory(inputFilename, workingDir);
+
+            if (!File.Exists(transcriptFile))
+            {
+                closed = true;
+                throw new Exception("Is not a transcript file. Unzipped to: " + workingDir);
+            }
+
+            return JsonConvert.DeserializeObject<OverwatchTranscript>(File.ReadAllText(transcriptFile))!;
+        }
+
+        private void CheckClosed()
+        {
+            if (closed) throw new Exception("Transcript has already been closed.");
+        }
+    }
+
+    public class ActivateMoment
+    {
+        public ActivateMoment(DateTime utc, TimeSpan? duration, long index)
+        {
+            Utc = utc;
+            Duration = duration;
+            Index = index;
+        }
+
+        public DateTime Utc { get; }
+        public TimeSpan? Duration { get; }
+        public long Index { get; }
+    }
+
+    public class ActivateEvent<T>
+    {
+        public ActivateEvent(ActivateMoment moment, T payload)
+        {
+            Moment = moment;
+            Payload = payload;
+        }
+
+        public ActivateMoment Moment { get; }
+        public T Payload { get; }
+    }
+}
@@ -0,0 +1,130 @@
+using Logging;
+using Newtonsoft.Json;
+using System.IO.Compression;
+
+namespace OverwatchTranscript
+{
+    public interface ITranscriptWriter
+    {
+        void AddHeader(string key, object value);
+        void Add(DateTime utc, object payload);
+        void IncludeArtifact(string filePath);
+        void Write(string outputFilename);
+    }
+
+    public class TranscriptWriter : ITranscriptWriter
+    {
+        private readonly object _lock = new object();
+        private readonly MomentReferenceBuilder builder;
+        private readonly string transcriptFile;
+        private readonly string artifactsFolder;
+        private readonly Dictionary<string, string> header = new Dictionary<string, string>();
+        private readonly BucketSet bucketSet;
+        private readonly ILog log;
+        private readonly string workingDir;
+        private bool closed;
+
+        public TranscriptWriter(ILog log, string workingDir)
+        {
+            closed = false;
+            this.log = log;
+            this.workingDir = workingDir;
+            bucketSet = new BucketSet(log, workingDir);
+            builder = new MomentReferenceBuilder(log, workingDir);
+            transcriptFile = Path.Combine(workingDir, TranscriptConstants.TranscriptFilename);
+            artifactsFolder = Path.Combine(workingDir, TranscriptConstants.ArtifactFolderName);
+
+            if (!Directory.Exists(workingDir)) Directory.CreateDirectory(workingDir);
+            if (File.Exists(transcriptFile) || Directory.Exists(artifactsFolder)) throw new Exception("workingdir not clean");
+        }
+
+        public void Add(DateTime utc, object payload)
+        {
+            CheckClosed();
+            bucketSet.Add(utc, payload);
+        }
+
+        public void AddHeader(string key, object value)
+        {
+            CheckClosed();
+            lock (_lock)
+            {
+                header.Add(key, Json.Serialize(value));
+            }
+        }
+
+        public void IncludeArtifact(string filePath)
+        {
+            CheckClosed();
+            if (!File.Exists(filePath)) throw new Exception("File not found: " + filePath);
+            if (!Directory.Exists(artifactsFolder)) Directory.CreateDirectory(artifactsFolder);
+            var name = Path.GetFileName(filePath);
+            File.Copy(filePath, Path.Combine(artifactsFolder, name), overwrite: false);
+        }
+
+        public void Write(string outputFilename)
+        {
+            CheckClosed();
+            closed = true;
+
+            var momentReferences = builder.Build(bucketSet.FinalizeBuckets());
+            var model = CreateModel(momentReferences);
+
+            File.WriteAllText(transcriptFile, Json.Serialize(model, Formatting.Indented));
+
+            ZipFile.CreateFromDirectory(workingDir, outputFilename);
+            log.Debug($"Transcript written to {outputFilename}");
+            log.Debug($"Common header: {Json.Serialize(model.Header.Common, Formatting.Indented)}");
+
+            Directory.Delete(workingDir, true);
+            log.Debug($"Workdir {workingDir} deleted");
+        }
+
+        private OverwatchTranscript CreateModel(OverwatchMomentReference[] momentReferences)
+        {
+            lock (_lock)
+            {
+                var model = new OverwatchTranscript
+                {
+                    Header = new OverwatchHeader
+                    {
+                        Common = CreateCommonHeader(momentReferences),
+                        Entries = header.Select(h =>
+                        {
+                            return new OverwatchHeaderEntry
+                            {
+                                Key = h.Key,
+                                Value = h.Value
+                            };
+                        }).ToArray()
+                    },
+                    MomentReferences = momentReferences
+                };
+
+                header.Clear();
+                return model;
+            }
+        }
+
+        private OverwatchCommonHeader CreateCommonHeader(OverwatchMomentReference[] momentReferences)
+        {
+            var moments = momentReferences.Sum(m => m.NumberOfMoments);
+            var events = momentReferences.Sum(m => m.NumberOfEvents);
+            var earliest = momentReferences.Min(m => m.EarliestUtc);
+            var latest = momentReferences.Max(m => m.LatestUtc);
+
+            return new OverwatchCommonHeader
+            {
+                NumberOfMoments = moments,
+                NumberOfEvents = events,
+                EarliestUtc = earliest,
+                LatestUtc = latest
+            };
+        }
+
+        private void CheckClosed()
+        {
+            if (closed) throw new Exception("Transcript has already been written. Cannot modify or write again.");
+        }
+    }
+}
@@ -2,12 +2,14 @@
 {
    public class Address
    {
-        public Address(string host, int port)
+        public Address(string logName, string host, int port)
        {
+            LogName = logName;
            Host = host;
            Port = port;
        }

+        public string LogName { get; }
        public string Host { get; }
        public int Port { get; }

@@ -20,5 +22,21 @@
        {
            return !string.IsNullOrEmpty(Host) && Port > 0;
        }
+
+        public static Address Empty()
+        {
+            return new Address(string.Empty, string.Empty, 0);
+        }
    }
+
+    public interface IHasMetricsScrapeTarget
+    {
+        Address GetMetricsScrapeTarget();
+    }
+    
+    public interface IHasManyMetricScrapeTargets
+    {
+        Address[] GetMetricsScrapeTargets();
+    }
+
 }
@@ -0,0 +1,64 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Utils
+{
+    public static class Base58
+    {
+        private const string Digits = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
+
+        public static string Encode(byte[] data)
+        {
+            // Decode byte[] to BigInteger
+            BigInteger intData = 0;
+            for (int i = 0; i < data.Length; i++)
+            {
+                intData = intData * 256 + data[i];
+            }
+
+            // Encode BigInteger to Base58 string
+            string result = "";
+            while (intData > 0)
+            {
+                int remainder = (int)(intData % 58);
+                intData /= 58;
+                result = Digits[remainder] + result;
+            }
+
+            // Append `1` for each leading 0 byte
+            for (int i = 0; i < data.Length && data[i] == 0; i++)
+            {
+                result = '1' + result;
+            }
+            return result;
+        }
+
+        public static byte[] Decode(string s)
+        {
+            BigInteger intData = 0;
+            for (int i = 0; i < s.Length; i++)
+            {
+                int digit = Digits.IndexOf(s[i]); //Slow
+                if (digit < 0)
+                    throw new FormatException(string.Format("Invalid Base58 character `{0}` at position {1}", s[i], i));
+                intData = intData * 58 + digit;
+            }
+
+            // Encode BigInteger to byte[]
+            // Leading zero bytes get encoded as leading `1` characters
+            int leadingZeroCount = s.TakeWhile(c => c == '1').Count();
+            var leadingZeros = Enumerable.Repeat((byte)0, leadingZeroCount);
+            var bytesWithoutLeadingZeros =
+                intData.ToByteArray()
+                .Reverse()// to big endian
+                .SkipWhile(b => b == 0);//strip sign byte
+            var result = leadingZeros.Concat(bytesWithoutLeadingZeros).ToArray();
+            return result;
+        }
+    }
+}
@@ -2,7 +2,7 @@
 {
    public class BlockInterval
    {
-        public BlockInterval(ulong from, ulong to)
+        public BlockInterval(TimeRange timeRange, ulong from, ulong to)
        {
            if (from < to)
            {
@@ -14,10 +14,14 @@
                From = to;
                To = from;
            }
+            TimeRange = timeRange;
+            NumberOfBlocks = (To - From) + 1;
        }

        public ulong From { get; }
        public ulong To { get; }
+        public TimeRange TimeRange { get; }
+        public ulong NumberOfBlocks { get; }

        public override string ToString()
        {
@@ -13,7 +13,6 @@

        public long SizeInBytes { get; }

-
        public long ToMB()
        {
            return SizeInBytes / (1024 * 1024);
@@ -0,0 +1,9 @@
+namespace Utils
+{
+    //public interface ICrashWatcher
+    //{
+    //    void Start();
+    //    void Stop();
+    //    bool HasCrashed();
+    //}
+}
@@ -0,0 +1,20 @@
+namespace Utils
+{
+    [Serializable]
+    public class EthAccount
+    {
+        public EthAccount(EthAddress ethAddress, string privateKey)
+        {
+            EthAddress = ethAddress;
+            PrivateKey = privateKey;
+        }
+
+        public EthAddress EthAddress { get; }
+        public string PrivateKey { get; }
+
+        public override string ToString()
+        {
+            return EthAddress.ToString();
+        }
+    }
+}
@@ -1,4 +1,4 @@
-namespace GethPlugin
+namespace Utils
 {
    public interface IHasEthAddress
    {
@@ -1,4 +1,4 @@
-namespace GethPlugin
+namespace Utils
 {
    public class Ether : IComparable<Ether>
    {
@@ -28,7 +28,13 @@

        public override string ToString()
        {
-            return $"{Eth} Eth";
+            var weiOnly = Wei % TokensIntExtensions.WeiPerEth;
+
+            var tokens = new List<string>();
+            if (Eth > 0) tokens.Add($"{Eth} Eth");
+            if (weiOnly > 0) tokens.Add($"{weiOnly} Wei");
+
+            return string.Join(" + ", tokens);
        }
    }

@@ -0,0 +1,41 @@
+namespace Utils
+{
+    public static class PluginPathUtils
+    {
+        private const string ProjectPluginsFolderName = "ProjectPlugins";
+        private static string projectPluginsDir = string.Empty;
+
+        public static string ProjectPluginsDir
+        {
+            get
+            {
+                if (string.IsNullOrEmpty(projectPluginsDir)) projectPluginsDir = FindProjectPluginsDir();
+                return projectPluginsDir;
+            }
+        }
+
+        private static string FindProjectPluginsDir()
+        {
+            var current = Directory.GetCurrentDirectory();
+            while (true)
+            {
+                var localFolders = Directory.GetDirectories(current);
+                var projectPluginsFolders = localFolders.Where(l => l.EndsWith(ProjectPluginsFolderName)).ToArray();
+                if (projectPluginsFolders.Length == 1)
+                {
+                    return projectPluginsFolders.Single();
+                }
+
+                var parent = Directory.GetParent(current);
+                if (parent == null)
+                {
+                    var msg = $"Unable to locate '{ProjectPluginsFolderName}' folder. Travelled up from: '{Directory.GetCurrentDirectory()}'";
+                    Console.WriteLine(msg);
+                    throw new Exception(msg);
+                }
+
+                current = parent.FullName;
+            }
+        }
+    }
+}
@@ -3,13 +3,41 @@
    public static class RandomUtils
    {
        private static readonly Random random = new Random();
+        private static readonly object @lock = new object();
+
+        public static T GetOneRandom<T>(this T[] items)
+        {
+            lock (@lock)
+            {
+                var i = random.Next(0, items.Length);
+                var result = items[i];
+                return result;
+            }
+        }

        public static T PickOneRandom<T>(this List<T> remainingItems)
        {
-            var i = random.Next(0, remainingItems.Count);
-            var result = remainingItems[i];
-            remainingItems.RemoveAt(i);
-            return result;
+            lock (@lock)
+            {
+                var i = random.Next(0, remainingItems.Count);
+                var result = remainingItems[i];
+                remainingItems.RemoveAt(i);
+                return result;
+            }
+        }
+
+        public static T[] Shuffled<T>(T[] items)
+        {
+            lock (@lock)
+            {
+                var result = new List<T>();
+                var source = items.ToList();
+                while (source.Any())
+                {
+                    result.Add(RandomUtils.PickOneRandom(source));
+                }
+                return result.ToArray();
+            }
        }
    }
 }
@@ -0,0 +1,135 @@
+namespace Utils
+{
+    public class Retry
+    {
+        private readonly string description;
+        private readonly TimeSpan maxTimeout;
+        private readonly TimeSpan sleepAfterFail;
+        private readonly Action<Failure> onFail;
+
+        public Retry(string description, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail)
+        {
+            this.description = description;
+            this.maxTimeout = maxTimeout;
+            this.sleepAfterFail = sleepAfterFail;
+            this.onFail = onFail;
+        }
+
+        public void Run(Action task)
+        {
+            var run = new RetryRun(description, task, maxTimeout, sleepAfterFail, onFail);
+            run.Run();
+        }
+
+        public T Run<T>(Func<T> task)
+        {
+            T? result = default;
+
+            var run = new RetryRun(description, () =>
+            {
+                result = task();
+            }, maxTimeout, sleepAfterFail, onFail);
+            run.Run();
+
+            return result!;
+        }
+
+        private class RetryRun
+        {
+            private readonly string description;
+            private readonly Action task;
+            private readonly TimeSpan maxTimeout;
+            private readonly TimeSpan sleepAfterFail;
+            private readonly Action<Failure> onFail;
+            private readonly DateTime start = DateTime.UtcNow;
+            private readonly List<Failure> failures = new List<Failure>();
+            private int tryNumber;
+            private DateTime tryStart;
+
+            public RetryRun(string description, Action task, TimeSpan maxTimeout, TimeSpan sleepAfterFail, Action<Failure> onFail)
+            {
+                this.description = description;
+                this.task = task;
+                this.maxTimeout = maxTimeout;
+                this.sleepAfterFail = sleepAfterFail;
+                this.onFail = onFail;
+
+                tryNumber = 0;
+                tryStart = DateTime.UtcNow;
+            }
+
+            public void Run()
+            {
+                while (true)
+                {
+                    CheckMaximums();
+
+                    tryNumber++;
+                    tryStart = DateTime.UtcNow;
+                    try
+                    {
+                        task();
+                        return;
+                    }
+                    catch (OperationCanceledException)
+                    {
+                        return;
+                    }
+                    catch (Exception ex)
+                    {
+                        var failure = CaptureFailure(ex);
+                        onFail(failure);
+                        Time.Sleep(sleepAfterFail);
+                    }
+                }
+            }
+
+            private Failure CaptureFailure(Exception ex)
+            {
+                var f = new Failure(ex, DateTime.UtcNow - tryStart, tryNumber);
+                failures.Add(f);
+                return f;
+            }
+
+            private void CheckMaximums()
+            {
+                if (Duration() > maxTimeout) Fail();
+            }
+
+            private void Fail()
+            {
+                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport()}",
+                        new AggregateException(failures.Select(f => f.Exception)));
+            }
+
+            private string GetFailureReport()
+            {
+                return Environment.NewLine + string.Join(Environment.NewLine, failures.Select(f => f.Describe()));
+            }
+
+            private TimeSpan Duration()
+            {
+                return DateTime.UtcNow - start;
+            }
+        }
+    }
+
+    public class Failure
+    {
+        public Failure(Exception exception, TimeSpan duration, int tryNumber)
+        {
+            Exception = exception;
+            Duration = duration;
+            TryNumber = tryNumber;
+        }
+
+        public Exception Exception { get; }
+        public TimeSpan Duration { get; }
+        public int TryNumber { get; }
+
+        public string Describe()
+        {
+            return $"Try {TryNumber} failed after {Time.FormatDuration(Duration)} with exception '{Exception}'";
+        }
+    }
+}
@@ -0,0 +1,28 @@
+namespace Utils
+{
+    public static class RollingAverage
+    {
+        /// <param name="currentAverage">Value of average before new value is added.</param>
+        /// <param name="newNumberOfValues">Number of values in average after new value is added.</param>
+        /// <param name="newValue">New value to be added.</param>
+        /// <returns>New average value.</returns>
+        /// <exception cref="Exception">newNumberOfValues must be 1 or greater.</exception>
+        public static float GetNewAverage(float currentAverage, int newNumberOfValues, float newValue)
+        {
+            if (newNumberOfValues < 1) throw new Exception("Should be at least 1 value.");
+
+            float n = newNumberOfValues;
+            var originalValue = currentAverage;
+            var originalValueWeight = ((n - 1.0f) / n);
+            var newValueWeight = (1.0f / n);
+            return GetWeightedAverage(originalValue, originalValueWeight, newValue, newValueWeight);
+        }
+
+        public static float GetWeightedAverage(float value1, float weight1, float value2, float weight2)
+        {
+            float totalWeight = weight1 + weight2;
+            if (totalWeight == 0.0f) return 0.0f;
+            return ((value1 * weight1) + (value2 * weight2)) / totalWeight;
+        }
+    }
+}
@@ -0,0 +1,13 @@
+namespace Utils
+{
+    public static class Str
+    {
+        public static string Between(string input, string open, string close)
+        {
+            var openIndex = input.IndexOf(open) + open.Length;
+            var closeIndex = input.LastIndexOf(close);
+
+            return input.Substring(openIndex, closeIndex - openIndex);
+        }
+    }
+}
@@ -0,0 +1,122 @@
+using System.Numerics;
+
+namespace Utils
+{
+    public class TestToken : IComparable<TestToken>
+    {
+        public static BigInteger WeiFactor = new BigInteger(1000000000000000000);
+
+        public TestToken(BigInteger tstWei)
+        {
+            TstWei = tstWei;
+            Tst = tstWei / WeiFactor;
+        }
+
+        public BigInteger TstWei { get; }
+        public BigInteger Tst { get; }
+
+        public int CompareTo(TestToken? other)
+        {
+            return TstWei.CompareTo(other!.TstWei);
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is TestToken token && TstWei == token.TstWei;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(TstWei);
+        }
+
+        public override string ToString()
+        {
+            var weiOnly = TstWei % WeiFactor;
+
+            var tokens = new List<string>();
+            if (Tst > 0) tokens.Add($"{Tst} TST");
+            if (weiOnly > 0) tokens.Add($"{weiOnly} TSTWEI");
+
+            return string.Join(" + ", tokens);
+        }
+
+        public static TestToken operator +(TestToken a, TestToken b)
+        {
+            return new TestToken(a.TstWei + b.TstWei);
+        }
+
+        public static TestToken operator -(TestToken a, TestToken b)
+        {
+            return new TestToken(a.TstWei - b.TstWei);
+        }
+
+        public static TestToken operator *(TestToken a, int b)
+        {
+            return new TestToken(a.TstWei * b);
+        }
+
+        public static bool operator <(TestToken a, TestToken b)
+        {
+            return a.TstWei < b.TstWei;
+        }
+
+        public static bool operator >(TestToken a, TestToken b)
+        {
+            return a.TstWei > b.TstWei;
+        }
+
+        public static bool operator ==(TestToken a, TestToken b)
+        {
+            return a.TstWei == b.TstWei;
+        }
+
+        public static bool operator !=(TestToken a, TestToken b)
+        {
+            return a.TstWei != b.TstWei;
+        }
+    }
+
+    public static class TestTokensExtensions
+    {
+        public static TestToken TstWei(this int i)
+        {
+            return new TestToken(new BigInteger(i));
+        }
+
+        public static TestToken TstWei(this decimal i)
+        {
+            return new TestToken(new BigInteger(i));
+        }
+
+        public static TestToken TstWei(this BigInteger i)
+        {
+            return new TestToken(i);
+        }
+
+        public static TestToken TstWei(this string s)
+        {
+            return new TestToken(BigInteger.Parse(s));
+        }
+
+        public static TestToken Tst(this int i)
+        {
+            return Tst(Convert.ToDecimal(i));
+        }
+
+        public static TestToken Tst(this decimal i)
+        {
+            return new TestToken(new BigInteger(i) * TestToken.WeiFactor);
+        }
+
+        public static TestToken Tst(this BigInteger i)
+        {
+            return new TestToken(i * TestToken.WeiFactor);
+        }
+
+        public static TestToken Tst(this string s)
+        {
+            return new TestToken(BigInteger.Parse(s) * TestToken.WeiFactor);
+        }
+    }
+}
@@ -1,6 +1,4 @@
-using System.Diagnostics;
-
-namespace Utils
+namespace Utils
 {
    public static class Time
    {
@@ -20,6 +18,12 @@ namespace Utils
            task.Wait();
        }

+        public static string FormatDuration(TimeSpan? d)
+        {
+            if (d == null) return "[NULL]";
+            return FormatDuration(d.Value);
+        }
+
        public static string FormatDuration(TimeSpan d)
        {
            var result = "";
@@ -29,6 +33,11 @@ namespace Utils
            result += $"{d.Seconds} secs";
            return result;
        }
+
+        public static string FormatTimestamp(DateTime d)
+        {
+            return d.ToString("o");
+        }
        
        public static TimeSpan ParseTimespan(string span)
        {
@@ -105,78 +114,24 @@ namespace Utils

        public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
        {
-            var start = DateTime.UtcNow;
-            var tries = 1;
-            var tryInfo = new List<(Exception, TimeSpan)>();
-
-            while (true)
-            {
-                var duration = DateTime.UtcNow - start;
-                if (duration > maxTimeout)
-                {
-                    var info = FormatTryInfos(tryInfo);
-                    throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.{Environment.NewLine}{info}");
-                }
-
-                var sw = Stopwatch.StartNew();
-                try
-                {
-                    action();
-                    return;
-                }
-                catch (Exception ex)
-                {
-                    tryInfo.Add((ex, sw.Elapsed));
-                    tries++;
-                }
-
-                Sleep(retryTime);
-            }
-        }
-
-        private static string FormatTryInfos(List<(Exception, TimeSpan)> tryInfo)
-        {
-            return string.Join(Environment.NewLine, tryInfo.Select(FormatTryInfo).ToArray());
-        }
-
-        private static string FormatTryInfo((Exception, TimeSpan) info, int index)
-        {
-            return $"Attempt {index} took {FormatDuration(info.Item2)} and failed with exception {info.Item1}.";
-        }
-
-        private static Action<int> failedCallback = i => { };
-        public static void SetRetryFailedCallback(Action<int> onRetryFailed)
-        {
-            failedCallback = onRetryFailed;
+            Retry(action, maxTimeout, retryTime, description, f => { });
        }

        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description)
        {
-            var start = DateTime.UtcNow;
-            var tries = 1;
-            var exceptions = new List<Exception>();
+            return Retry(action, maxTimeout, retryTime, description, f => { });
+        }

-            while (true)
-            {
-                var duration = DateTime.UtcNow - start;
-                if (duration > maxTimeout)
-                {
-                    throw new TimeoutException($"Retry '{description}' timed out after {tries} tries over {FormatDuration(duration)}.", new AggregateException(exceptions));
-                }
+        public static void Retry(Action action, TimeSpan maxTimeout, TimeSpan retryTime, string description, Action<Failure> onFail)
+        {
+            var r = new Retry(description, maxTimeout, retryTime, onFail);
+            r.Run(action);
+        }

-                try
-                {
-                    return action();
-                }
-                catch (Exception ex)
-                {
-                    exceptions.Add(ex);
-                    failedCallback(tries);
-                    tries++;
-                }
-
-                Sleep(retryTime);
-            }
+        public static T Retry<T>(Func<T> action, TimeSpan maxTimeout, TimeSpan retryTime, string description, Action<Failure> onFail)
+        {
+            var r = new Retry(description, maxTimeout, retryTime, onFail);
+            return r.Run(action);
        }
    }
 }
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
 	  <RootNamespace>Utils</RootNamespace>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
@@ -1,11 +1,10 @@
-using Logging;
-using Newtonsoft.Json;
-using Serialization = Newtonsoft.Json.Serialization;
-using System.Net.Http.Headers;
+using System.Net.Http.Headers;
 using System.Net.Http.Json;
+using Logging;
+using Newtonsoft.Json;
 using Utils;

-namespace Core
+namespace WebUtils
 {
    public interface IEndpoint
    {
@@ -119,7 +118,7 @@ namespace Core
            var errors = new List<string>();
            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
            {
-                Error = delegate (object? sender, Serialization.ErrorEventArgs args)
+                Error = delegate (object? sender, Newtonsoft.Json.Serialization.ErrorEventArgs args)
                {
                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
                    {
@@ -1,29 +1,33 @@
 using Logging;
 using Utils;

-namespace Core
+namespace WebUtils
 {
    public interface IHttp
    {
        T OnClient<T>(Func<HttpClient, T> action);
        T OnClient<T>(Func<HttpClient, T> action, string description);
+        T OnClient<T>(Func<HttpClient, T> action, Retry retry);
        IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null);
    }

    internal class Http : IHttp
    {
-        private static readonly object httpLock = new object();
+        private static object lockLock = new object();
+        private static readonly Dictionary<string, object> httpLocks = new Dictionary<string, object>();
        private readonly ILog log;
-        private readonly ITimeSet timeSet;
+        private readonly IWebCallTimeSet timeSet;
        private readonly Action<HttpClient> onClientCreated;
+        private readonly string id;

-        internal Http(ILog log, ITimeSet timeSet)
-            : this(log, timeSet, DoNothing)
+        internal Http(string id, ILog log, IWebCallTimeSet timeSet)
+            : this(id, log, timeSet, DoNothing)
        {
        }

-        internal Http(ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
+        internal Http(string id, ILog log, IWebCallTimeSet timeSet, Action<HttpClient> onClientCreated)
        {
+            this.id = id;
            this.log = log;
            this.timeSet = timeSet;
            this.onClientCreated = onClientCreated;
@@ -35,13 +39,19 @@ namespace Core
        }

        public T OnClient<T>(Func<HttpClient, T> action, string description)
+        {
+            var retry = new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), f => { });
+            return OnClient(action, retry);
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action, Retry retry)
        {
            var client = GetClient();

            return LockRetry(() =>
            {
                return action(client);
-            }, description);
+            }, retry);
        }

        public IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null)
@@ -54,11 +64,21 @@ namespace Core
            return DebugStack.GetCallerName(skipFrames: 2);
        }

-        private T LockRetry<T>(Func<T> operation, string description)
+        private T LockRetry<T>(Func<T> operation, Retry retry)
        {
+            var httpLock = GetLock();
            lock (httpLock)
            {
-                return Time.Retry(operation, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), description);
+                return retry.Run(operation);
+            }
+        }
+
+        private object GetLock()
+        {
+            lock (lockLock) // I had to.
+            {
+                if (!httpLocks.ContainsKey(id)) httpLocks.Add(id, new object());
+                return httpLocks[id];
            }
        }

@@ -0,0 +1,43 @@
+using Logging;
+
+namespace WebUtils
+{
+    public interface IHttpFactory
+    {
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet timeSet);
+        IHttp CreateHttp(string id);
+    }
+
+    public class HttpFactory : IHttpFactory
+    {
+        private readonly ILog log;
+        private readonly IWebCallTimeSet defaultTimeSet;
+
+        public HttpFactory(ILog log)
+            : this (log, new DefaultWebCallTimeSet())
+        {
+        }
+
+        public HttpFactory(ILog log, IWebCallTimeSet defaultTimeSet)
+        {
+            this.log = log;
+            this.defaultTimeSet = defaultTimeSet;
+        }
+
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
+        {
+            return CreateHttp(id, onClientCreated, defaultTimeSet);
+        }
+
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet ts)
+        {
+            return new Http(id, log, ts, onClientCreated);
+        }
+
+        public IHttp CreateHttp(string id)
+        {
+            return new Http(id, log, defaultTimeSet);
+        }
+    }
+}
@@ -1,6 +1,6 @@
-namespace Core
+namespace WebUtils
 {
-    public interface ITimeSet
+    public interface IWebCallTimeSet
    {
        /// <summary>
        /// Timeout for a single HTTP call.
@@ -17,48 +17,27 @@
        /// After a failed HTTP call, wait this long before trying again.
        /// </summary>
        TimeSpan HttpCallRetryDelay();
-
-        /// <summary>
-        /// After a failed K8s operation, wait this long before trying again.
-        /// </summary>
-        TimeSpan K8sOperationRetryDelay();
-
-        /// <summary>
-        /// Maximum total time to attempt to perform a successful k8s operation.
-        /// If k8s operations fail during this timespan, retries will be made.
-        /// </summary>
-        TimeSpan K8sOperationTimeout();
    }

-    public class DefaultTimeSet : ITimeSet
+    public class DefaultWebCallTimeSet : IWebCallTimeSet
    {
        public TimeSpan HttpCallTimeout()
        {
-            return TimeSpan.FromMinutes(3);
+            return TimeSpan.FromMinutes(2);
        }

        public TimeSpan HttpRetryTimeout()
        {
-            return TimeSpan.FromMinutes(10);
+            return TimeSpan.FromMinutes(5);
        }

        public TimeSpan HttpCallRetryDelay()
        {
            return TimeSpan.FromSeconds(1);
        }
-
-        public TimeSpan K8sOperationRetryDelay()
-        {
-            return TimeSpan.FromSeconds(10);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromMinutes(30);
-        }
    }

-    public class LongTimeSet : ITimeSet
+    public class LongWebCallTimeSet : IWebCallTimeSet
    {
        public TimeSpan HttpCallTimeout()
        {
@@ -74,15 +53,5 @@
        {
            return TimeSpan.FromSeconds(20);
        }
-
-        public TimeSpan K8sOperationRetryDelay()
-        {
-            return TimeSpan.FromSeconds(30);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromHours(1);
-        }
    }
 }
@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+	<ItemGroup>
+		<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+	</ItemGroup>
+
+	<ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+    <ProjectReference Include="..\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>
--- a/Show More
+++ b/Show More