Fixes too-frequent save of overview json. Resets failure count on upload.

# Conflicts:
#	Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs

.github/workflows/docker-autoclient.yml

@@ -1,5 +1,6 @@
 name: Docker - AutoClient
 
+
 on:
   push:
     branches:
@@ -12,15 +13,15 @@ on:
       - 'Framework/**'
       - 'ProjectPlugins/**'
       - .github/workflows/docker-autoclient.yml
-      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
+
 jobs:
   build-and-push:
     name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
     with:
       docker_file: Tools/AutoClient/docker/Dockerfile
-      docker_repo: codexstorage/codex-autoclient
+      dockerhub_repo: codexstorage/codex-autoclient
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
     secrets: inherit
-

.github/workflows/docker-discordbot.yml

@@ -13,16 +13,15 @@ on:
       - 'Framework/**'
       - 'ProjectPlugins/**'
       - .github/workflows/docker-discordbot.yml
-      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
 
 jobs:
   build-and-push:
     name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
     with:
       docker_file: Tools/BiblioTech/docker/Dockerfile
-      docker_repo: codexstorage/codex-discordbot
+      dockerhub_repo: codexstorage/codex-discordbot
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
     secrets: inherit
-

.github/workflows/docker-keymaker.yml

@@ -11,17 +11,16 @@ on:
       - 'Tools/KeyMaker/**'
       - 'Framework/**'
       - 'ProjectPlugins/**'
-      - .github/workflows/docker-KeyMaker.yml
-      - .github/workflows/docker-reusable.yml
+      - .github/workflows/docker-keymaker.yml
   workflow_dispatch:
 
 
 jobs:
   build-and-push:
     name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
     with:
       docker_file: Tools/KeyMaker/docker/Dockerfile
-      docker_repo: codexstorage/codex-keymaker
+      dockerhub_repo: codexstorage/codex-keymaker
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
     secrets: inherit
-

.github/workflows/docker-marketinsights.yml

@@ -12,16 +12,15 @@ on:
       - 'Framework/**'
       - 'ProjectPlugins/**'
       - .github/workflows/docker-marketinsights.yml
-      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
 
 jobs:
   build-and-push:
     name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
     with:
       docker_file: Tools/MarketInsights/Dockerfile
-      docker_repo: codexstorage/codex-marketinsights
+      dockerhub_repo: codexstorage/codex-marketinsights
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
     secrets: inherit
-

.github/workflows/docker-reusable.yml

@@ -1,178 +0,0 @@
-name: Reusable - Docker
-
-
-on:
-  workflow_call:
-    inputs:
-      docker_file:
-        default: docker/Dockerfile
-        description: Dockerfile
-        required: false
-        type: string
-      docker_repo:
-        default: codexstorage/cs-codex-dist-tests
-        description: DockerHub repository
-        required: false
-        type: string
-      tag_latest:
-        default: true
-        description: Set latest tag for Docker images
-        required: false
-        type: boolean
-      tag_sha:
-        default: true
-        description: Set Git short commit as Docker tag
-        required: false
-        type: boolean
-      tag_suffix:
-        default: ''
-        description: Suffix for Docker images tag
-        required: false
-        type: string
-
-
-env:
-  DOCKER_FILE: ${{ inputs.docker_file }}
-  DOCKER_REPO: ${{ inputs.docker_repo }}
-  TAG_LATEST: ${{ inputs.tag_latest }}
-  TAG_SHA: ${{ inputs.tag_sha }}
-  TAG_SUFFIX: ${{ inputs.tag_suffix }}
-
-
-jobs:
-  # Build platform specific image
-  build:
-    strategy:
-      fail-fast: true
-      matrix:
-        target:
-          - os: linux
-            arch: amd64
-          - os: linux
-            arch: arm64
-        include:
-          - target:
-              os: linux
-              arch: amd64
-            builder: ubuntu-22.04
-          - target:
-              os: linux
-              arch: arm64
-            builder: buildjet-4vcpu-ubuntu-2204-arm
-
-    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
-    runs-on: ${{ matrix.builder }}
-    env:
-      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.DOCKER_REPO }}
-
-      - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Build and Push by digest
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ${{ env.DOCKER_FILE }}
-          platforms: ${{ env.PLATFORM }}
-          push: true
-          labels: ${{ steps.meta.outputs.labels }}
-          outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
-
-      - name: Docker - Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-      - name: Docker - Upload digest
-        uses: actions/upload-artifact@v4
-        with:
-          name: digests-${{ matrix.target.arch }}
-          path: /tmp/digests
-          if-no-files-found: error
-          retention-days: 1
-
-
-  # Publish multi-platform image
-  publish:
-    name: Publish multi-platform image
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Docker - Variables
-        run: |
-          # Adjust custom suffix when set and
-          if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
-            echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
-          fi
-          # Disable SHA tags on tagged release
-          if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
-            echo "TAG_SHA=false" >>$GITHUB_ENV
-          fi
-          # Handle latest and latest-custom using raw
-          if [[ ${{ env.TAG_SHA }} == "false" ]]; then
-            echo "TAG_LATEST=false" >>$GITHUB_ENV
-            echo "TAG_RAW=true" >>$GITHUB_ENV
-            if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
-              echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
-            else
-              echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
-            fi
-          else
-            echo "TAG_RAW=false" >>$GITHUB_ENV
-          fi
-
-      - name: Docker - Download digests
-        uses: actions/download-artifact@v4
-        with:
-          pattern: digests-*
-          merge-multiple: true
-          path: /tmp/digests
-
-      - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.DOCKER_REPO }}
-          flavor: |
-            latest=${{ env.TAG_LATEST }}
-            suffix=${{ env.TAG_SUFFIX }},onlatest=true
-          tags: |
-            type=semver,pattern={{version}}
-            type=raw,enable=${{ env.TAG_RAW }},value=latest
-            type=sha,enable=${{ env.TAG_SHA }}
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Create manifest list and push
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
-
-      - name: Docker - Inspect image
-        run: |
-          docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}

.github/workflows/docker-rewarder.yml

@@ -1,5 +1,6 @@
 name: Docker - Rewarder Bot
 
+
 on:
   push:
     branches:
@@ -12,15 +13,15 @@ on:
       - 'Framework/**'
       - 'ProjectPlugins/**'
       - .github/workflows/docker-rewarder.yml
-      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
+
 jobs:
   build-and-push:
     name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
     with:
       docker_file: Tools/TestNetRewarder/docker/Dockerfile
-      docker_repo: codexstorage/codex-rewarderbot
+      dockerhub_repo: codexstorage/codex-rewarderbot
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
     secrets: inherit
-

.github/workflows/docker-runner.yml

@@ -11,12 +11,15 @@ on:
       - docker/Dockerfile
       - docker/docker-entrypoint.sh
       - .github/workflows/docker-runner.yml
-      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
 
 jobs:
   build-and-push:
     name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
+    uses: codex-storage/github-actions/.github/workflows/docker-reusable.yml@master
+    with:
+      docker_file: docker/Dockerfile
+      dockerhub_repo: codexstorage/cs-codex-dist-tests
+      tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
     secrets: inherit

.github/workflows/run-continuous-tests.yaml

@@ -80,7 +80,7 @@ env:
   TESTS_TARGET_DURATION: 2d
   TESTS_FILTER: ""
   TESTS_CLEANUP: true
-  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  JOB_MANIFEST: docker/job-continuous-tests.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
   KUBE_VERSION: v1.28.2
 
@@ -146,7 +146,7 @@ jobs:
         if: false
         run: |
           # Variables
-          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to run a node
           duration=600
           namespace="${{ env.NAMESPACE }}"
           pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")

.github/workflows/run-dist-tests.yaml

@@ -32,7 +32,7 @@ env:
   NAMEPREFIX: d-tests-runner
   NAMESPACE: default
   COMMAND: dotnet test Tests/CodexTests
-  JOB_MANIFEST: docker/dist-tests-job.yaml
+  JOB_MANIFEST: docker/job-dist-tests.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
   KUBE_VERSION: v1.28.2
 

.github/workflows/run-release-tests.yaml

@@ -0,0 +1,122 @@
+name: Run Release Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: "Codex Docker image (example: 'codexstorage/nim-codex:0.1.8-dist-tests')"
+        required: true
+        type: string
+      workflow_source:
+        description: Workflow source
+        required: false
+        type: string
+        default: ''
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
+  TEST_TYPE: release-tests
+  NAMEPREFIX: r-tests
+  NAMESPACE: default
+  JOB_MANIFEST: docker/job-release-tests.yaml
+  COMMAND: dotnet test Tests/CodexReleaseTests
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.30.5
+
+
+jobs:
+  run_tests:
+    name: Run Release Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
+          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'${COMMAND}'"')" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job to run tests
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}"
+          echo "Runner pod: $(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}')"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+
+      - name: Show Runner logs
+        run: |
+          # Variables
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          namespace="${{ env.NAMESPACE }}"
+          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath='{.items[0].metadata.name}')
+
+          # Check Pod status
+          WAIT=120
+          SECONDS=0
+          sleep=1
+          while (( SECONDS < WAIT )); do
+            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}')
+            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in the $phase state - Get the logs"; break; } || { echo "Pod $pod is in the $phase state - Retry in $sleep second(s) / $((WAIT - SECONDS))"; }
+            sleep $sleep
+          done
+
+          # Get logs
+          while [[ $(kubectl get pod ${pod} -n ${namespace} -o jsonpath='{.status.phase}') == "Running" ]]; do
+            echo "Show ${pod} logs ..."
+            echo "----"
+            kubectl logs $pod -n $namespace -f || true
+            sleep 1
+          done
+
+      - name: Set workflow status from job status
+        run: |
+          sleep 5
+          job_status=$(kubectl get jobs ${{ env.NAMEPREFIX }} -n ${{ env.NAMESPACE }} -o jsonpath='{.status.conditions[0].type}')
+          echo "Job status: $job_status"
+          if [[ "${job_status}" != "Complete" ]]; then exit 1; fi

.gitignore

@@ -3,3 +3,4 @@ obj
 bin
 .vscode
 Tools/AutoClient/datapath
+.editorconfig

Framework/BlockchainUtils/BlockCache.cs

@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockCache
     {

Framework/BlockchainUtils/BlockTimeEntry.cs

@@ -1,4 +1,4 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockTimeEntry
     {

Framework/BlockchainUtils/BlockTimeFinder.cs

@@ -1,6 +1,6 @@
 using Logging;
 
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
     public class BlockTimeFinder
     {

Framework/BlockchainUtils/BlockchainBounds.cs

@@ -1,5 +1,11 @@
-namespace NethereumWorkflow.BlockUtils
+namespace BlockchainUtils
 {
+    public interface IWeb3Blocks
+    {
+        ulong GetCurrentBlockNumber();
+        DateTime? GetTimestampForBlock(ulong blockNumber);
+    }
+
     public class BlockchainBounds
     {
         private readonly BlockCache cache;

Framework/BlockchainUtils/BlockchainUtils.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+	<ItemGroup>
+		<PackageReference Include="Nethereum.Web3" Version="4.14.0" />
+	</ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/BlockchainUtils/ConversionExtensions.cs

@@ -1,7 +1,7 @@
 using Nethereum.Hex.HexTypes;
 using System.Numerics;
 
-namespace NethereumWorkflow
+namespace BlockchainUtils
 {
     public static class ConversionExtensions
     {

Framework/Core/Core.csproj

@@ -9,6 +9,7 @@
   <ItemGroup>
     <ProjectReference Include="..\FileUtils\FileUtils.csproj" />
     <ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
+    <ProjectReference Include="..\WebUtils\WebUtils.csproj" />
   </ItemGroup>
 
 </Project>

Framework/Core/CoreInterface.cs

@@ -1,5 +1,6 @@
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
+using Logging;
 
 namespace Core
 {

Framework/Core/EntryPoint.cs

@@ -1,5 +1,6 @@
 using KubernetesWorkflow;
 using Logging;
+using WebUtils;
 
 namespace Core
 {
@@ -8,16 +9,16 @@ namespace Core
         private readonly IToolsFactory toolsFactory;
         private readonly PluginManager manager = new PluginManager();
 
-        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
         {
-            toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, timeSet);
+            toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, webCallTimeSet, k8STimeSet);
 
             Tools = toolsFactory.CreateTools();
             manager.InstantiatePlugins(PluginFinder.GetPluginTypes(), toolsFactory);
         }
 
         public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder)
-            : this(log, configuration, fileManagerRootFolder, new DefaultTimeSet())
+            : this(log, configuration, fileManagerRootFolder, new DefaultWebCallTimeSet(), new DefaultK8sTimeSet())
         {
         }
 

Framework/Core/PluginTools.cs

@@ -1,12 +1,14 @@
 using FileUtils;
 using KubernetesWorkflow;
 using Logging;
+using WebUtils;
 
 namespace Core
 {
-    public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
+    public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactory, IFileTool
     {
-        ITimeSet TimeSet { get; }
+        IWebCallTimeSet WebCallTimeSet { get; }
+        IK8sTimeSet K8STimeSet { get; }
 
         /// <summary>
         /// Deletes kubernetes and tracked file resources.
@@ -25,13 +27,6 @@ namespace Core
         ILog GetLog();
     }
 
-    public interface IHttpFactoryTool
-    {
-        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
-        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet timeSet);
-        IHttp CreateHttp(string id);
-    }
-
     public interface IFileTool
     {
         IFileManager GetFileManager();
@@ -40,18 +35,22 @@ namespace Core
     internal class PluginTools : IPluginTools
     {
         private readonly WorkflowCreator workflowCreator;
+        private readonly HttpFactory httpFactory;
         private readonly IFileManager fileManager;
         private readonly LogPrefixer log;
 
-        internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet)
+        internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
         {
             this.log = new LogPrefixer(log);
             this.workflowCreator = workflowCreator;
-            TimeSet = timeSet;
+            httpFactory = new HttpFactory(log, webCallTimeSet);
+            WebCallTimeSet = webCallTimeSet;
+            K8STimeSet = k8STimeSet;
             fileManager = new FileManager(log, fileManagerRootFolder);
         }
 
-        public ITimeSet TimeSet { get; }
+        public IWebCallTimeSet WebCallTimeSet { get; }
+        public IK8sTimeSet K8STimeSet { get; }
 
         public void ApplyLogPrefix(string prefix)
         {
@@ -60,17 +59,17 @@ namespace Core
 
         public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
         {
-            return CreateHttp(id, onClientCreated, TimeSet);
+            return httpFactory.CreateHttp(id, onClientCreated);
         }
 
-        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet ts)
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet timeSet)
         {
-            return new Http(id, log, ts, onClientCreated);
+            return httpFactory.CreateHttp(id, onClientCreated, timeSet);
         }
 
         public IHttp CreateHttp(string id)
         {
-            return new Http(id, log, TimeSet);
+            return httpFactory.CreateHttp(id);
         }
 
         public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)

Framework/Core/ToolsFactory.cs

@@ -1,5 +1,6 @@
 using KubernetesWorkflow;
 using Logging;
+using WebUtils;
 
 namespace Core
 {
@@ -13,19 +14,21 @@ namespace Core
         private readonly ILog log;
         private readonly WorkflowCreator workflowCreator;
         private readonly string fileManagerRootFolder;
-        private readonly ITimeSet timeSet;
+        private readonly IWebCallTimeSet webCallTimeSet;
+        private readonly IK8sTimeSet k8STimeSet;
 
-        public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, IWebCallTimeSet webCallTimeSet, IK8sTimeSet k8STimeSet)
         {
             this.log = log;
             workflowCreator = new WorkflowCreator(log, configuration);
             this.fileManagerRootFolder = fileManagerRootFolder;
-            this.timeSet = timeSet;
+            this.webCallTimeSet = webCallTimeSet;
+            this.k8STimeSet = k8STimeSet;
         }
 
         public PluginTools CreateTools()
         {
-            return new PluginTools(log, workflowCreator, fileManagerRootFolder, timeSet);
+            return new PluginTools(log, workflowCreator, fileManagerRootFolder, webCallTimeSet, k8STimeSet);
         }
     }
 }

Framework/FileUtils/TrackedFile.cs

@@ -14,6 +14,12 @@ namespace FileUtils
             Label = label;
         }
 
+        public static TrackedFile FromPath(ILog log, string filepath)
+        {
+            // todo: I don't wanne have to do this to call upload.
+            return new TrackedFile(log, filepath, string.Empty);
+        }
+
         public string Filename { get; }
         public string Label { get; }
 

Framework/GethConnector/GethConnector.cs

@@ -1,4 +1,6 @@
-using CodexContractsPlugin;
+using BlockchainUtils;
+using CodexContractsPlugin;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 
@@ -18,18 +20,29 @@ namespace GethConnector
                 return null;
             }
 
+            var gethNode = new CustomGethNode(log, new BlockCache(), GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+
+            var config = GetCodexMarketplaceConfig(gethNode, GethInput.MarketplaceAddress);
+
             var contractsDeployment = new CodexContractsDeployment(
+                config: config,
                 marketplaceAddress: GethInput.MarketplaceAddress,
                 abi: GethInput.ABI,
                 tokenAddress: GethInput.TokenAddress
             );
 
-            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
             var contracts = new CodexContractsAccess(log, gethNode, contractsDeployment);
 
             return new GethConnector(gethNode, contracts);
         }
 
+        private static MarketplaceConfig GetCodexMarketplaceConfig(IGethNode gethNode, string marketplaceAddress)
+        {
+            var func = new ConfigurationFunctionBase();
+            var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
+            return response.ReturnValue1;
+        }
+
         private GethConnector(IGethNode gethNode, ICodexContracts codexContracts)
         {
             GethNode = gethNode;

Framework/GethConnector/GethInput.cs

@@ -45,8 +45,12 @@
         private static string? GetEnvVar(List<string> error, string name)
         {
             var result = Environment.GetEnvironmentVariable(name);
-            if (string.IsNullOrEmpty(result)) error.Add($"'{name}' is not set.");
-            return result;
+            if (string.IsNullOrEmpty(result))
+            {
+                error.Add($"'{name}' is not set.");
+                return null;
+            }
+            return result.Trim();
         }
     }
 }

Framework/KubernetesWorkflow/ContainerCrashWatcher.cs

@@ -3,7 +3,7 @@ using Logging;
 
 namespace KubernetesWorkflow
 {
-    public class CrashWatcher
+    public class ContainerCrashWatcher
     {
         private readonly ILog log;
         private readonly KubernetesClientConfiguration config;
@@ -15,7 +15,7 @@ namespace KubernetesWorkflow
         private Task? worker;
         private Exception? workerException;
 
-        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
+        public ContainerCrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
         {
             this.log = log;
             this.config = config;
@@ -45,7 +45,7 @@ namespace KubernetesWorkflow
             if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
         }
 
-        public bool HasContainerCrashed()
+        public bool HasCrashed()
         {
             using var client = new Kubernetes(config);
             var result = HasContainerBeenRestarted(client);
@@ -70,13 +70,13 @@ namespace KubernetesWorkflow
             using var client = new Kubernetes(config);
             while (!token.IsCancellationRequested)
             {
-                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10));
-
                 if (HasContainerBeenRestarted(client))
                 {
                     DownloadCrashedContainerLogs(client);
                     return;
                 }
+
+                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10));
             }
         }
 
@@ -91,7 +91,7 @@ namespace KubernetesWorkflow
         private void DownloadCrashedContainerLogs(Kubernetes client)
         {
             using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
-            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
+            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName, containerName);
             handler.Log(stream);
         }
     }

Framework/KubernetesWorkflow/DownloadedLog.cs

@@ -1,93 +0,0 @@
-using Logging;
-
-namespace KubernetesWorkflow
-{
-    public interface IDownloadedLog
-    {
-        string ContainerName { get; }
-
-        void IterateLines(Action<string> action, params string[] thatContain);
-        string[] GetLinesContaining(string expectedString);
-        string[] FindLinesThatContain(params string[] tags);
-        string GetFilepath();
-        void DeleteFile();
-    }
-
-    internal class DownloadedLog : IDownloadedLog
-    {
-        private readonly LogFile logFile;
-
-        internal DownloadedLog(WriteToFileLogHandler logHandler, string containerName)
-        {
-            logFile = logHandler.LogFile;
-            ContainerName = containerName;
-        }
-
-        public string ContainerName { get; }
-
-        public void IterateLines(Action<string> action, params string[] thatContain)
-        {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (thatContain.All(line.Contains))
-                {
-                    action(line);
-                }
-                line = streamReader.ReadLine();
-            }
-        }
-
-        public string[] GetLinesContaining(string expectedString)
-        {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-            var lines = new List<string>();
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (line.Contains(expectedString))
-                {
-                    lines.Add(line);
-                }
-                line = streamReader.ReadLine();
-            }
-
-            return lines.ToArray(); ;
-        }
-
-        public string[] FindLinesThatContain(params string[] tags)
-        {
-            var result = new List<string>();
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (tags.All(line.Contains))
-                {
-                    result.Add(line);
-                }
-
-                line = streamReader.ReadLine();
-            }
-
-            return result.ToArray();
-        }
-
-        public string GetFilepath()
-        {
-            return logFile.FullFilename;
-        }
-
-        public void DeleteFile()
-        {
-            File.Delete(logFile.FullFilename);
-        }
-    }
-}

Framework/KubernetesWorkflow/K8sController.cs

@@ -122,6 +122,16 @@ namespace KubernetesWorkflow
             var all = client.Run(c => c.ListNamespace().Items);
             var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(prefix));
 
+            if (wait)
+            {
+                // If we're going to wait, trigger the delete for all the namespaces immediately.
+                // Then wait for them to finish one by one.
+                foreach (var ns in namespaces)
+                {
+                    DeleteNamespace(ns, false);
+                }
+            }
+
             foreach (var ns in namespaces)
             {
                 DeleteNamespace(ns, wait);
@@ -735,7 +745,10 @@ namespace KubernetesWorkflow
                 throw new Exception($"Expected to find 1 pod by podLabel '{deployment.PodLabel}'. Found: {pods.Length}. " +
                     $"Total number of pods: {allPods.Items.Count}. Their labels: {string.Join(Environment.NewLine, allLabels)}");
             }
-            return pods[0];
+            var pod = pods[0];
+            if (pod.Status == null) throw new Exception("Pod status unknown");
+            if (string.IsNullOrEmpty(pod.Status.PodIP)) throw new Exception("Pod IP unknown");
+            return pod;
         }
 
         #endregion
@@ -906,7 +919,7 @@ namespace KubernetesWorkflow
                 var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
                 log.Error(msg);
 
-                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg, deploymentName), tailLines: null, previous: true);
 
                 throw new Exception(msg);
             }
@@ -946,13 +959,13 @@ namespace KubernetesWorkflow
 
         #endregion
 
-        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        public ContainerCrashWatcher CreateCrashWatcher(RunningContainer container)
         {
             var containerName = container.Name;
             var podName = GetPodName(container);
             var recipeName = container.Recipe.Name;
 
-            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
+            return new ContainerCrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
         }
 
         private V1Pod[] FindPodsByLabel(string podLabel)

Framework/KubernetesWorkflow/K8sTimeSet.cs

@@ -0,0 +1,42 @@
+namespace Core
+{
+    public interface IK8sTimeSet
+    {
+        /// <summary>
+        /// After a failed K8s operation, wait this long before trying again.
+        /// </summary>
+        TimeSpan K8sOperationRetryDelay();
+
+        /// <summary>
+        /// Maximum total time to attempt to perform a successful k8s operation.
+        /// If k8s operations fail during this timespan, retries will be made.
+        /// </summary>
+        TimeSpan K8sOperationTimeout();
+    }
+
+    public class DefaultK8sTimeSet : IK8sTimeSet
+    {
+        public TimeSpan K8sOperationRetryDelay()
+        {
+            return TimeSpan.FromSeconds(10);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromMinutes(30);
+        }
+    }
+
+    public class LongK8sTimeSet : IK8sTimeSet
+    {
+        public TimeSpan K8sOperationRetryDelay()
+        {
+            return TimeSpan.FromSeconds(30);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromHours(1);
+        }
+    }
+}

Framework/KubernetesWorkflow/LogHandler.cs

@@ -25,11 +25,11 @@ namespace KubernetesWorkflow
 
     public class WriteToFileLogHandler : LogHandler, ILogHandler
     {
-        public WriteToFileLogHandler(ILog sourceLog, string description)
+        public WriteToFileLogHandler(ILog sourceLog, string description, string addFileName)
         {
-            LogFile = sourceLog.CreateSubfile();
+            LogFile = sourceLog.CreateSubfile(addFileName);
 
-            var msg = $"{description} -->> {LogFile.FullFilename}";
+            var msg = $"{description} -->> {LogFile.Filename}";
             sourceLog.Log(msg);
 
             LogFile.Write(msg);

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -13,7 +13,7 @@ namespace KubernetesWorkflow
         FutureContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
         PodInfo GetPodInfo(RunningContainer container);
         PodInfo GetPodInfo(RunningPod pod);
-        CrashWatcher CreateCrashWatcher(RunningContainer container);
+        ContainerCrashWatcher CreateCrashWatcher(RunningContainer container);
         void Stop(RunningPod pod, bool waitTillStopped);
         void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null, bool? previous = null);
         IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null, bool? previous = null);
@@ -61,7 +61,8 @@ namespace KubernetesWorkflow
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
-                var rc = new RunningPod(Guid.NewGuid().ToString(), startupConfig, startResult, containers);
+                var info = GetPodInfo(startResult.Deployment);
+                var rc = new RunningPod(Guid.NewGuid().ToString(), info, startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
 
                 if (startResult.ExternalService != null)
@@ -83,9 +84,14 @@ namespace KubernetesWorkflow
             });
         }
 
+        public PodInfo GetPodInfo(RunningDeployment deployment)
+        {
+            return K8s(c => c.GetPodInfo(deployment));
+        }
+
         public PodInfo GetPodInfo(RunningContainer container)
         {
-            return K8s(c => c.GetPodInfo(container.RunningPod.StartResult.Deployment));
+            return GetPodInfo(container.RunningPod.StartResult.Deployment);
         }
 
         public PodInfo GetPodInfo(RunningPod pod)
@@ -93,7 +99,7 @@ namespace KubernetesWorkflow
             return K8s(c => c.GetPodInfo(pod.StartResult.Deployment));
         }
 
-        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        public ContainerCrashWatcher CreateCrashWatcher(RunningContainer container)
         {
             return K8s(c => c.CreateCrashWatcher(container));
         }
@@ -127,14 +133,14 @@ namespace KubernetesWorkflow
         {
             var msg = $"Downloading container log for '{container.Name}'";
             log.Log(msg);
-            var logHandler = new WriteToFileLogHandler(log, msg);
+            var logHandler = new WriteToFileLogHandler(log, msg, container.Name);
 
             K8s(controller =>
             {
                 controller.DownloadPodLog(container, logHandler, tailLines, previous);
             });
 
-            return new DownloadedLog(logHandler, container.Name);
+            return new DownloadedLog(logHandler.LogFile, container.Name);
         }
 
         public string ExecuteCommand(RunningContainer container, string command, params string[] args)
@@ -209,6 +215,7 @@ namespace KubernetesWorkflow
             var port = startResult.GetExternalServicePorts(recipe, tag);
 
             return new Address(
+                logName: $"{recipe.Name}:{tag}",
                 startResult.Cluster.HostAddress,
                 port.Number);
         }
@@ -220,6 +227,7 @@ namespace KubernetesWorkflow
             var port = startResult.GetInternalServicePorts(recipe, tag);
 
             return new Address(
+                logName: $"{serviceName}:{tag}",
                 $"http://{serviceName}.{namespaceName}.svc.cluster.local",
                 port.Number);
         }

Framework/KubernetesWorkflow/Types/RunningPod.cs

@@ -4,9 +4,10 @@ namespace KubernetesWorkflow.Types
 {
     public class RunningPod
     {
-        public RunningPod(string id, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        public RunningPod(string id, PodInfo podInfo, StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
         {
             Id = id;
+            PodInfo = podInfo;
             StartupConfig = startupConfig;
             StartResult = startResult;
             Containers = containers;
@@ -15,6 +16,7 @@ namespace KubernetesWorkflow.Types
         }
 
         public string Id { get; }
+        public PodInfo PodInfo { get; }
         public StartupConfig StartupConfig { get; }
         public StartResult StartResult { get; }
         public RunningContainer[] Containers { get; }

Framework/Logging/BaseLog.cs

@@ -8,7 +8,7 @@ namespace Logging
         void Debug(string message = "", int skipFrames = 0);
         void Error(string message);
         void AddStringReplace(string from, string to);
-        LogFile CreateSubfile(string ext = "log");
+        LogFile CreateSubfile(string addName, string ext = "log");
     }
 
     public abstract class BaseLog : ILog
@@ -34,7 +34,7 @@ namespace Logging
         {
             get
             {
-                if (logFile == null) logFile = new LogFile(GetFullName(), "log");
+                if (logFile == null) logFile = new LogFile(GetFullName() + ".log");
                 return logFile;
             }
         }
@@ -69,12 +69,16 @@ namespace Logging
 
         public virtual void Delete()
         {
-            File.Delete(LogFile.FullFilename);
+            File.Delete(LogFile.Filename);
         }
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
+            addName = addName
+                .Replace("<", "")
+                .Replace(">", "");
+
+            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}_{addName}.{ext}");
         }
 
         protected string ApplyReplacements(string str)

Framework/Logging/DownloadedLog.cs

@@ -0,0 +1,79 @@
+namespace Logging
+{
+    public interface IDownloadedLog
+    {
+        string SourceName { get; }
+
+        void IterateLines(Action<string> action);
+        void IterateLines(Action<string> action, params string[] thatContain);
+        string[] GetLinesContaining(string expectedString);
+        string[] FindLinesThatContain(params string[] tags);
+        string GetFilepath();
+        void DeleteFile();
+    }
+
+    public class DownloadedLog : IDownloadedLog
+    {
+        private readonly LogFile logFile;
+
+        public DownloadedLog(string filepath, string sourceName)
+        {
+            logFile = new LogFile(filepath);
+            SourceName = sourceName;
+        }
+
+        public DownloadedLog(LogFile logFile, string sourceName)
+        {
+            this.logFile = logFile;
+            SourceName = sourceName;
+        }
+
+        public string SourceName { get; }
+
+        public void IterateLines(Action<string> action)
+        {
+            using var file = File.OpenRead(logFile.Filename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                action(line);
+                line = streamReader.ReadLine();
+            }
+        }
+
+        public void IterateLines(Action<string> action, params string[] thatContain)
+        {
+            IterateLines(line =>
+            {
+                if (thatContain.All(line.Contains))
+                {
+                    action(line);
+                }
+            });
+        }
+
+        public string[] GetLinesContaining(string expectedString)
+        {
+            return FindLinesThatContain([expectedString]);
+        }
+
+        public string[] FindLinesThatContain(params string[] tags)
+        {
+            var result = new List<string>();
+            IterateLines(result.Add, tags);
+            return result.ToArray();
+        }
+
+        public string GetFilepath()
+        {
+            return logFile.Filename;
+        }
+
+        public void DeleteFile()
+        {
+            File.Delete(logFile.Filename);
+        }
+    }
+}

Framework/Logging/LogFile.cs

@@ -1,21 +1,19 @@
-namespace Logging
+using Utils;
+
+namespace Logging
 {
     public class LogFile
     {
-        private readonly string extension;
         private readonly object fileLock = new object();
-        private string filename;
 
-        public LogFile(string filename, string extension)
+        public LogFile(string filename)
         {
-            this.filename = filename;
-            this.extension = extension;
-            FullFilename = filename + "." + extension;
+            Filename = filename;
 
             EnsurePathExists(filename);
         }
 
-        public string FullFilename { get; private set; }
+        public string Filename { get; private set; }
 
         public void Write(string message)
         {
@@ -28,7 +26,7 @@
             {
                 lock (fileLock)
                 { 
-                    File.AppendAllLines(FullFilename, new[] { message });
+                    File.AppendAllLines(Filename, new[] { message });
                 }
             }
             catch (Exception ex)
@@ -37,19 +35,24 @@
             }
         }
 
-        public void ConcatToFilename(string toAdd)
+        public void WriteRawMany(IEnumerable<string> lines)
         {
-            var oldFullName = FullFilename;
-
-            filename += toAdd;
-            FullFilename = filename + "." + extension;
-
-            File.Move(oldFullName, FullFilename);
+            try
+            {
+                lock (fileLock)
+                {
+                    File.AppendAllLines(Filename, lines);
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine("Writing to log has failed: " + ex);
+            }
         }
 
         private static string GetTimestamp()
         {
-            return $"[{DateTime.UtcNow.ToString("o")}]";
+            return $"[{Time.FormatTimestamp(DateTime.UtcNow)}]";
         }
 
         private void EnsurePathExists(string filename)

Framework/Logging/LogPrefixer.cs

@@ -18,9 +18,9 @@
         public string Prefix { get; set; } = string.Empty;
 
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return backingLog.CreateSubfile(ext);
+            return backingLog.CreateSubfile(addName, ext);
         }
 
         public void Debug(string message = "", int skipFrames = 0)

Framework/Logging/LogSplitter.cs

@@ -14,9 +14,9 @@
             OnAll(l => l.AddStringReplace(from, to));
         }
 
-        public LogFile CreateSubfile(string ext = "log")
+        public LogFile CreateSubfile(string addName, string ext = "log")
         {
-            return targetLogs.First().CreateSubfile(ext);
+            return targetLogs.First().CreateSubfile(addName, ext);
         }
 
         public void Debug(string message = "", int skipFrames = 0)

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -1,25 +1,25 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace NethereumWorkflow
 {
     public class NethereumInteraction
     {
-        // BlockCache is a static instance: It stays alive for the duration of the application runtime.
-        private readonly static BlockCache blockCache = new BlockCache();
+        private readonly BlockCache blockCache;
 
         private readonly ILog log;
         private readonly Web3 web3;
 
-        internal NethereumInteraction(ILog log, Web3 web3)
+        internal NethereumInteraction(ILog log, Web3 web3, BlockCache blockCache)
         {
             this.log = log;
             this.web3 = web3;
+            this.blockCache = blockCache;
         }
 
         public string SendEth(string toAddress, decimal ethAmount)
@@ -50,6 +50,21 @@ namespace NethereumWorkflow
             return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function));
         }
 
+        public TResult Call<TFunction, TResult>(string contractAddress, TFunction function, ulong blockNumber) where TFunction : FunctionMessage, new()
+        {
+            log.Debug(typeof(TFunction).ToString());
+            var handler = web3.Eth.GetContractQueryHandler<TFunction>();
+            return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function, new BlockParameter(blockNumber)));
+        }
+
+        public void Call<TFunction>(string contractAddress, TFunction function, ulong blockNumber) where TFunction : FunctionMessage, new()
+        {
+            log.Debug(typeof(TFunction).ToString());
+            var handler = web3.Eth.GetContractQueryHandler<TFunction>();
+            var result = Time.Wait(handler.QueryRawAsync(contractAddress, function, new BlockParameter(blockNumber)));
+            var aaaa = 0;
+        }
+
         public string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
         {
             log.Debug();

Framework/NethereumWorkflow/NethereumInteractionCreator.cs

@@ -1,4 +1,5 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.Web3;
 
 namespace NethereumWorkflow
@@ -6,13 +7,15 @@ namespace NethereumWorkflow
     public class NethereumInteractionCreator
     {
         private readonly ILog log;
+        private readonly BlockCache blockCache;
         private readonly string ip;
         private readonly int port;
         private readonly string privateKey;
 
-        public NethereumInteractionCreator(ILog log, string ip, int port, string privateKey)
+        public NethereumInteractionCreator(ILog log, BlockCache blockCache, string ip, int port, string privateKey)
         {
             this.log = log;
+            this.blockCache = blockCache;
             this.ip = ip;
             this.port = port;
             this.privateKey = privateKey;
@@ -21,7 +24,7 @@ namespace NethereumWorkflow
         public NethereumInteraction CreateWorkflow()
         {
             log.Debug("Starting interaction to " + ip + ":" + port);
-            return new NethereumInteraction(log, CreateWeb3());
+            return new NethereumInteraction(log, CreateWeb3(), blockCache);
         }
 
         private Web3 CreateWeb3()

Framework/NethereumWorkflow/NethereumWorkflow.csproj

@@ -12,6 +12,7 @@
 	</ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\BlockchainUtils\BlockchainUtils.csproj" />
     <ProjectReference Include="..\Logging\Logging.csproj" />
     <ProjectReference Include="..\Utils\Utils.csproj" />
   </ItemGroup>

Framework/NethereumWorkflow/Web3Wrapper.cs

@@ -1,16 +1,11 @@
-using Logging;
+using BlockchainUtils;
+using Logging;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
 using Utils;
 
 namespace NethereumWorkflow
 {
-    public interface IWeb3Blocks
-    {
-        ulong GetCurrentBlockNumber();
-        DateTime? GetTimestampForBlock(ulong blockNumber);
-    }
-
     public class Web3Wrapper : IWeb3Blocks
     {
         private readonly Web3 web3;

Framework/Utils/Address.cs

@@ -2,12 +2,14 @@
 {
     public class Address
     {
-        public Address(string host, int port)
+        public Address(string logName, string host, int port)
         {
+            LogName = logName;
             Host = host;
             Port = port;
         }
 
+        public string LogName { get; }
         public string Host { get; }
         public int Port { get; }
 
@@ -20,5 +22,21 @@
         {
             return !string.IsNullOrEmpty(Host) && Port > 0;
         }
+
+        public static Address Empty()
+        {
+            return new Address(string.Empty, string.Empty, 0);
+        }
     }
+
+    public interface IHasMetricsScrapeTarget
+    {
+        Address GetMetricsScrapeTarget();
+    }
+    
+    public interface IHasManyMetricScrapeTargets
+    {
+        Address[] GetMetricsScrapeTargets();
+    }
+
 }

Framework/Utils/Base58.cs

@@ -0,0 +1,64 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Contracts;
+using System.Linq;
+using System.Numerics;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Utils
+{
+    public static class Base58
+    {
+        private const string Digits = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
+
+        public static string Encode(byte[] data)
+        {
+            // Decode byte[] to BigInteger
+            BigInteger intData = 0;
+            for (int i = 0; i < data.Length; i++)
+            {
+                intData = intData * 256 + data[i];
+            }
+
+            // Encode BigInteger to Base58 string
+            string result = "";
+            while (intData > 0)
+            {
+                int remainder = (int)(intData % 58);
+                intData /= 58;
+                result = Digits[remainder] + result;
+            }
+
+            // Append `1` for each leading 0 byte
+            for (int i = 0; i < data.Length && data[i] == 0; i++)
+            {
+                result = '1' + result;
+            }
+            return result;
+        }
+
+        public static byte[] Decode(string s)
+        {
+            BigInteger intData = 0;
+            for (int i = 0; i < s.Length; i++)
+            {
+                int digit = Digits.IndexOf(s[i]); //Slow
+                if (digit < 0)
+                    throw new FormatException(string.Format("Invalid Base58 character `{0}` at position {1}", s[i], i));
+                intData = intData * 58 + digit;
+            }
+
+            // Encode BigInteger to byte[]
+            // Leading zero bytes get encoded as leading `1` characters
+            int leadingZeroCount = s.TakeWhile(c => c == '1').Count();
+            var leadingZeros = Enumerable.Repeat((byte)0, leadingZeroCount);
+            var bytesWithoutLeadingZeros =
+                intData.ToByteArray()
+                .Reverse()// to big endian
+                .SkipWhile(b => b == 0);//strip sign byte
+            var result = leadingZeros.Concat(bytesWithoutLeadingZeros).ToArray();
+            return result;
+        }
+    }
+}

Framework/Utils/CrashWatcher.cs

@@ -0,0 +1,9 @@
+namespace Utils
+{
+    //public interface ICrashWatcher
+    //{
+    //    void Start();
+    //    void Stop();
+    //    bool HasCrashed();
+    //}
+}

Framework/Utils/EthAccount.cs

@@ -0,0 +1,20 @@
+namespace Utils
+{
+    [Serializable]
+    public class EthAccount
+    {
+        public EthAccount(EthAddress ethAddress, string privateKey)
+        {
+            EthAddress = ethAddress;
+            PrivateKey = privateKey;
+        }
+
+        public EthAddress EthAddress { get; }
+        public string PrivateKey { get; }
+
+        public override string ToString()
+        {
+            return EthAddress.ToString();
+        }
+    }
+}

Framework/Utils/EthAddress.cs

@@ -1,4 +1,4 @@
-namespace GethPlugin
+namespace Utils
 {
     public interface IHasEthAddress
     {

Framework/Utils/EthTokenExtensions.cs

@@ -1,4 +1,4 @@
-namespace GethPlugin
+namespace Utils
 {
     public class Ether : IComparable<Ether>
     {

Framework/Utils/Retry.cs

@@ -98,7 +98,7 @@
 
             private void Fail()
             {
-                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport}",
+                throw new TimeoutException($"Retry '{description}' timed out after {tryNumber} tries over {Time.FormatDuration(Duration())}: {GetFailureReport()}",
                         new AggregateException(failures.Select(f => f.Exception)));
             }
 

Framework/Utils/TestTokenExtensions.cs

@@ -1,6 +1,6 @@
 using System.Numerics;
 
-namespace CodexContractsPlugin
+namespace Utils
 {
     public class TestToken : IComparable<TestToken>
     {
@@ -46,6 +46,16 @@ namespace CodexContractsPlugin
             return new TestToken(a.TstWei + b.TstWei);
         }
 
+        public static TestToken operator -(TestToken a, TestToken b)
+        {
+            return new TestToken(a.TstWei - b.TstWei);
+        }
+
+        public static TestToken operator *(TestToken a, int b)
+        {
+            return new TestToken(a.TstWei * b);
+        }
+
         public static bool operator <(TestToken a, TestToken b)
         {
             return a.TstWei < b.TstWei;
@@ -71,7 +81,7 @@ namespace CodexContractsPlugin
     {
         public static TestToken TstWei(this int i)
         {
-            return TstWei(Convert.ToDecimal(i));
+            return new TestToken(new BigInteger(i));
         }
 
         public static TestToken TstWei(this decimal i)
@@ -84,6 +94,11 @@ namespace CodexContractsPlugin
             return new TestToken(i);
         }
 
+        public static TestToken TstWei(this string s)
+        {
+            return new TestToken(BigInteger.Parse(s));
+        }
+
         public static TestToken Tst(this int i)
         {
             return Tst(Convert.ToDecimal(i));
@@ -98,5 +113,10 @@ namespace CodexContractsPlugin
         {
             return new TestToken(i * TestToken.WeiFactor);
         }
+
+        public static TestToken Tst(this string s)
+        {
+            return new TestToken(BigInteger.Parse(s) * TestToken.WeiFactor);
+        }
     }
 }

Framework/Utils/Time.cs

@@ -33,6 +33,11 @@
             result += $"{d.Seconds} secs";
             return result;
         }
+
+        public static string FormatTimestamp(DateTime d)
+        {
+            return d.ToString("o");
+        }
         
         public static TimeSpan ParseTimespan(string span)
         {

Framework/WebUtils/Endpoint.cs

@@ -1,11 +1,10 @@
-using Logging;
-using Newtonsoft.Json;
-using Serialization = Newtonsoft.Json.Serialization;
-using System.Net.Http.Headers;
+using System.Net.Http.Headers;
 using System.Net.Http.Json;
+using Logging;
+using Newtonsoft.Json;
 using Utils;
 
-namespace Core
+namespace WebUtils
 {
     public interface IEndpoint
     {
@@ -119,7 +118,7 @@ namespace Core
             var errors = new List<string>();
             var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
             {
-                Error = delegate (object? sender, Serialization.ErrorEventArgs args)
+                Error = delegate (object? sender, Newtonsoft.Json.Serialization.ErrorEventArgs args)
                 {
                     if (args.CurrentObject == args.ErrorContext.OriginalObject)
                     {

Framework/WebUtils/Http.cs

@@ -1,7 +1,7 @@
 using Logging;
 using Utils;
 
-namespace Core
+namespace WebUtils
 {
     public interface IHttp
     {
@@ -16,16 +16,16 @@ namespace Core
         private static object lockLock = new object();
         private static readonly Dictionary<string, object> httpLocks = new Dictionary<string, object>();
         private readonly ILog log;
-        private readonly ITimeSet timeSet;
+        private readonly IWebCallTimeSet timeSet;
         private readonly Action<HttpClient> onClientCreated;
         private readonly string id;
 
-        internal Http(string id, ILog log, ITimeSet timeSet)
+        internal Http(string id, ILog log, IWebCallTimeSet timeSet)
             : this(id, log, timeSet, DoNothing)
         {
         }
 
-        internal Http(string id, ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
+        internal Http(string id, ILog log, IWebCallTimeSet timeSet, Action<HttpClient> onClientCreated)
         {
             this.id = id;
             this.log = log;

Framework/WebUtils/HttpFactory.cs

@@ -0,0 +1,43 @@
+using Logging;
+
+namespace WebUtils
+{
+    public interface IHttpFactory
+    {
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet timeSet);
+        IHttp CreateHttp(string id);
+    }
+
+    public class HttpFactory : IHttpFactory
+    {
+        private readonly ILog log;
+        private readonly IWebCallTimeSet defaultTimeSet;
+
+        public HttpFactory(ILog log)
+            : this (log, new DefaultWebCallTimeSet())
+        {
+        }
+
+        public HttpFactory(ILog log, IWebCallTimeSet defaultTimeSet)
+        {
+            this.log = log;
+            this.defaultTimeSet = defaultTimeSet;
+        }
+
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
+        {
+            return CreateHttp(id, onClientCreated, defaultTimeSet);
+        }
+
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, IWebCallTimeSet ts)
+        {
+            return new Http(id, log, ts, onClientCreated);
+        }
+
+        public IHttp CreateHttp(string id)
+        {
+            return new Http(id, log, defaultTimeSet);
+        }
+    }
+}

Framework/WebUtils/WebCallTimeSet.cs

@@ -1,6 +1,6 @@
-namespace Core
+namespace WebUtils
 {
-    public interface ITimeSet
+    public interface IWebCallTimeSet
     {
         /// <summary>
         /// Timeout for a single HTTP call.
@@ -17,20 +17,9 @@
         /// After a failed HTTP call, wait this long before trying again.
         /// </summary>
         TimeSpan HttpCallRetryDelay();
-
-        /// <summary>
-        /// After a failed K8s operation, wait this long before trying again.
-        /// </summary>
-        TimeSpan K8sOperationRetryDelay();
-
-        /// <summary>
-        /// Maximum total time to attempt to perform a successful k8s operation.
-        /// If k8s operations fail during this timespan, retries will be made.
-        /// </summary>
-        TimeSpan K8sOperationTimeout();
     }
 
-    public class DefaultTimeSet : ITimeSet
+    public class DefaultWebCallTimeSet : IWebCallTimeSet
     {
         public TimeSpan HttpCallTimeout()
         {
@@ -46,19 +35,9 @@
         {
             return TimeSpan.FromSeconds(1);
         }
-
-        public TimeSpan K8sOperationRetryDelay()
-        {
-            return TimeSpan.FromSeconds(10);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromMinutes(30);
-        }
     }
 
-    public class LongTimeSet : ITimeSet
+    public class LongWebCallTimeSet : IWebCallTimeSet
     {
         public TimeSpan HttpCallTimeout()
         {
@@ -74,15 +53,5 @@
         {
             return TimeSpan.FromSeconds(20);
         }
-
-        public TimeSpan K8sOperationRetryDelay()
-        {
-            return TimeSpan.FromSeconds(30);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromHours(1);
-        }
     }
 }

Framework/WebUtils/WebUtils.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+	<ItemGroup>
+		<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+	</ItemGroup>
+
+	<ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+    <ProjectReference Include="..\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>

ProjectPlugins/CodexClient/CodexAccess.cs

@@ -0,0 +1,314 @@
+using CodexOpenApi;
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+using WebUtils;
+
+namespace CodexClient
+{
+    public class CodexAccess
+    {
+        private readonly ILog log;
+        private readonly IHttpFactory httpFactory;
+        private readonly IProcessControl processControl;
+        private ICodexInstance instance;
+        private readonly Mapper mapper = new Mapper();
+
+        public CodexAccess(ILog log, IHttpFactory httpFactory, IProcessControl processControl, ICodexInstance instance)
+        {
+            this.log = log;
+            this.httpFactory = httpFactory;
+            this.processControl = processControl;
+            this.instance = instance;
+        }
+
+        public void Stop(bool waitTillStopped)
+        {
+            processControl.Stop(waitTillStopped);
+            // Prevents accidental use after stop:
+            instance = null!;
+        }
+
+        public IDownloadedLog DownloadLog(string additionalName = "")
+        {
+            var file = log.CreateSubfile(GetName() + additionalName);
+            Log($"Downloading logs to '{file.Filename}'");
+            return processControl.DownloadLog(file);
+        }
+
+        public string GetImageName()
+        {
+            return instance.ImageName;
+        }
+
+        public DateTime GetStartUtc()
+        {
+            return instance.StartUtc;
+        }
+
+        public DebugInfo GetDebugInfo()
+        {
+            return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
+        }
+
+        public void SetLogLevel(string logLevel)
+        {
+            try
+            {
+                OnCodex(async api =>
+                {
+                    await api.SetDebugLogLevelAsync(logLevel);
+                    return string.Empty;
+                });
+            }
+            catch (Exception exc)
+            {
+                log.Error("Failed to set log level: " + exc);
+            }
+        }
+
+        public string GetSpr()
+        {
+            return CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                var json = endpoint.HttpGetString("spr");
+                var response = JsonConvert.DeserializeObject<SprResponse>(json);
+                return response!.Spr;
+            });
+        }
+
+        private class SprResponse
+        {
+            public string Spr { get; set; } = string.Empty;
+        }
+
+        public DebugPeer GetDebugPeer(string peerId)
+        {
+            // Cannot use openAPI: debug/peer endpoint is not specified there.
+            return CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                var str = endpoint.HttpGetString($"debug/peer/{peerId}");
+
+                if (str.ToLowerInvariant() == "unable to find peer!")
+                {
+                    return new DebugPeer
+                    {
+                        IsPeerFound = false
+                    };
+                }
+
+                var result = endpoint.Deserialize<DebugPeer>(str);
+                result.IsPeerFound = true;
+                return result;
+            });
+        }
+
+        public void ConnectToPeer(string peerId, string[] peerMultiAddresses)
+        {
+            OnCodex(api =>
+            {
+                Time.Wait(api.ConnectPeerAsync(peerId, peerMultiAddresses));
+                return Task.FromResult(string.Empty);
+            });
+        }
+
+        public string UploadFile(UploadInput uploadInput)
+        {
+            return OnCodex(api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream));
+        }
+
+        public Stream DownloadFile(string contentId)
+        {
+            var fileResponse = OnCodex(api => api.DownloadNetworkStreamAsync(contentId));
+            if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
+            return fileResponse.Stream;
+        }
+
+        public LocalDataset DownloadStreamless(ContentId cid)
+        {
+            var response = OnCodex(api => api.DownloadNetworkAsync(cid.Id));
+            return mapper.Map(response);
+        }
+
+        public LocalDataset DownloadManifestOnly(ContentId cid)
+        {
+            var response = OnCodex(api => api.DownloadNetworkManifestAsync(cid.Id));
+            return mapper.Map(response);
+        }
+
+        public LocalDatasetList LocalFiles()
+        {
+            return mapper.Map(OnCodex(api => api.ListDataAsync()));
+        }
+
+        public StorageAvailability SalesAvailability(StorageAvailability request)
+        {
+            var body = mapper.Map(request);
+            var read = OnCodex(api => api.OfferStorageAsync(body));
+            return mapper.Map(read);
+        }
+
+        public StorageAvailability[] GetAvailabilities()
+        {
+            var collection = OnCodex(api => api.GetAvailabilitiesAsync());
+            return mapper.Map(collection);
+        }
+
+        public string RequestStorage(StoragePurchaseRequest request)
+        {
+            var body = mapper.Map(request);
+            return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
+        }
+
+        public CodexSpace Space()
+        {
+            var space = OnCodex(api => api.SpaceAsync());
+            return mapper.Map(space);
+        }
+
+        public StoragePurchase? GetPurchaseStatus(string purchaseId)
+        {
+            return CrashCheck(() =>
+            {
+                var endpoint = GetEndpoint();
+                try
+                {
+                    return Time.Retry(() =>
+                    {
+                        var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
+                        if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
+                        return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
+                    }, nameof(GetPurchaseStatus));
+                }
+                catch (Exception exc)
+                {
+                    log.Error($"Failed to fetch purchase information for id: '{purchaseId}'. Exception: {exc.Message}");
+                    return null;
+                }
+            });
+
+            // TODO: current getpurchase api does not line up with its openapi spec.
+            // return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
+        }
+
+        public string GetName()
+        {
+            return instance.Name;
+        }
+
+        public Address GetDiscoveryEndpoint()
+        {
+            return instance.DiscoveryEndpoint;
+        }
+
+        public Address GetApiEndpoint()
+        {
+            return instance.ApiEndpoint;
+        }
+
+        public Address GetListenEndpoint()
+        {
+            return instance.ListenEndpoint;
+        }
+
+        public bool HasCrashed()
+        {
+            return processControl.HasCrashed();
+        }
+
+        public Address? GetMetricsEndpoint()
+        {
+            return instance.MetricsEndpoint;
+        }
+
+        public EthAccount? GetEthAccount()
+        {
+            return instance.EthAccount;
+        }
+
+        public void DeleteDataDirFolder()
+        {
+            processControl.DeleteDataDirFolder();
+        }
+
+        private T OnCodex<T>(Func<CodexApiClient, Task<T>> action)
+        {
+            var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action));
+            return result;
+        }
+
+        private T OnCodex<T>(Func<CodexApiClient, Task<T>> action, Retry retry)
+        {
+            var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action), retry);
+            return result;
+        }
+
+        private T CallCodex<T>(HttpClient client, Func<CodexApiClient, Task<T>> action)
+        {
+            var address = GetAddress();
+            var api = new CodexApiClient(client);
+            api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
+            return CrashCheck(() => Time.Wait(action(api)));
+        }
+
+        private T CrashCheck<T>(Func<T> action)
+        {
+            try
+            {
+                return action();
+            }
+            finally
+            {
+                CheckContainerCrashed();
+            }
+        }
+
+        private IEndpoint GetEndpoint()
+        {
+            return httpFactory
+                .CreateHttp(GetHttpId(), h => CheckContainerCrashed())
+                .CreateEndpoint(GetAddress(), "/api/codex/v1/", GetName());
+        }
+
+        private Address GetAddress()
+        {
+            return instance.ApiEndpoint;
+        }
+
+        private string GetHttpId()
+        {
+            return GetAddress().ToString();
+        }
+
+        private void CheckContainerCrashed()
+        {
+            if (processControl.HasCrashed()) throw new Exception($"Container {GetName()} has crashed.");
+        }
+
+        private void Throw(Failure failure)
+        {
+            throw failure.Exception;
+        }
+
+        private void Log(string msg)
+        {
+            log.Log($"({GetName()}) {msg}");
+        }
+    }
+
+    public class UploadInput
+    {
+        public UploadInput(string contentType, string contentDisposition, FileStream fileStream)
+        {
+            ContentType = contentType;
+            ContentDisposition = contentDisposition;
+            FileStream = fileStream;
+        }
+
+        public string ContentType { get; }
+        public string ContentDisposition { get; }
+        public FileStream FileStream { get; }
+    }
+}

ProjectPlugins/CodexClient/CodexClient.csproj

@@ -0,0 +1,35 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Remove="openapi.yaml" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <OpenApiReference Include="openapi.yaml" CodeGenerator="NSwagCSharp" Namespace="CodexOpenApi" ClassName="CodexApiClient" />
+  </ItemGroup>
+
+<ItemGroup>
+	<PackageReference Include="Microsoft.Extensions.ApiDescription.Client" Version="7.0.2">
+		<PrivateAssets>all</PrivateAssets>
+		<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+	</PackageReference>
+	<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+	<PackageReference Include="NSwag.ApiDescription.Client" Version="13.18.2">
+		<PrivateAssets>all</PrivateAssets>
+		<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+	</PackageReference>
+</ItemGroup>
+	
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\FileUtils\FileUtils.csproj" />
+    <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
+    <ProjectReference Include="..\..\Framework\WebUtils\WebUtils.csproj" />
+  </ItemGroup>
+
+</Project>

ProjectPlugins/CodexClient/CodexInstance.cs

@@ -0,0 +1,54 @@
+using Utils;
+
+namespace CodexClient
+{
+    public interface ICodexInstance
+    {
+        string Name { get; }
+        string ImageName { get; }
+        DateTime StartUtc { get; }
+        Address DiscoveryEndpoint { get; }
+        Address ApiEndpoint { get; }
+        Address ListenEndpoint { get; }
+        EthAccount? EthAccount { get; }
+        Address? MetricsEndpoint { get; }
+    }
+
+    public class CodexInstance : ICodexInstance
+    {
+        public CodexInstance(string name, string imageName, DateTime startUtc, Address discoveryEndpoint, Address apiEndpoint, Address listenEndpoint, EthAccount? ethAccount, Address? metricsEndpoint)
+        {
+            Name = name;
+            ImageName = imageName;
+            StartUtc = startUtc;
+            DiscoveryEndpoint = discoveryEndpoint;
+            ApiEndpoint = apiEndpoint;
+            ListenEndpoint = listenEndpoint;
+            EthAccount = ethAccount;
+            MetricsEndpoint = metricsEndpoint;
+        }
+
+        public string Name { get; }
+        public string ImageName { get; }
+        public DateTime StartUtc { get; }
+        public Address DiscoveryEndpoint { get; }
+        public Address ApiEndpoint { get; }
+        public Address ListenEndpoint { get; }
+        public EthAccount? EthAccount { get; }
+        public Address? MetricsEndpoint { get; }
+
+        public static ICodexInstance CreateFromApiEndpoint(string name, Address apiEndpoint, EthAccount? ethAccount = null)
+        {
+            return new CodexInstance(
+                name,
+                imageName: "-",
+                startUtc: DateTime.UtcNow,
+                discoveryEndpoint: Address.Empty(),
+                apiEndpoint: apiEndpoint,
+                listenEndpoint: Address.Empty(),
+                ethAccount: ethAccount,
+                metricsEndpoint: null
+            );
+        }
+    }
+}

ProjectPlugins/CodexClient/CodexLogLevel.cs

@@ -1,4 +1,4 @@
-namespace CodexPlugin
+namespace CodexClient
 {
     public enum CodexLogLevel
     {

ProjectPlugins/CodexClient/CodexLogLine.cs

@@ -1,6 +1,6 @@
 using System.Globalization;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class CodexLogLine
     {

ProjectPlugins/CodexClient/CodexNode.cs

@@ -0,0 +1,409 @@
+using CodexClient.Hooks;
+using FileUtils;
+using Logging;
+using Utils;
+
+namespace CodexClient
+{
+    public partial interface ICodexNode : IHasEthAddress, IHasMetricsScrapeTarget
+    {
+        string GetName();
+        string GetImageName();
+        string GetPeerId();
+        DebugInfo GetDebugInfo(bool log = false);
+        void SetLogLevel(string logLevel);
+        string GetSpr();
+        DebugPeer GetDebugPeer(string peerId);
+        ContentId UploadFile(TrackedFile file);
+        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition);
+        TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
+        LocalDataset DownloadStreamless(ContentId cid);
+        /// <summary>
+        /// TODO: This will monitor the quota-used of the node until 'size' bytes are added. That's a very bad way
+        /// to track the streamless download progress. Replace it once we have a good API for this.
+        /// </summary>
+        LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size);
+        LocalDataset DownloadManifestOnly(ContentId cid);
+        LocalDatasetList LocalFiles();
+        CodexSpace Space();
+        void ConnectToPeer(ICodexNode node);
+        DebugInfoVersion Version { get; }
+        IMarketplaceAccess Marketplace { get; }
+        ITransferSpeeds TransferSpeeds { get; }
+        EthAccount EthAccount { get; }
+        StoragePurchase? GetPurchaseStatus(string purchaseId);
+
+        Address GetDiscoveryEndpoint();
+        Address GetApiEndpoint();
+        Address GetListenEndpoint();
+
+        /// <summary>
+        /// Warning! The node is not usable after this.
+        /// TODO: Replace with delete-blocks debug call once available in Codex.
+        /// </summary>
+        void DeleteDataDirFolder();
+        void Stop(bool waitTillStopped);
+        IDownloadedLog DownloadLog(string additionalName = "");
+        bool HasCrashed();
+    }
+
+    public class CodexNode : ICodexNode
+    {
+        private const string UploadFailedMessage = "Unable to store block";
+        private readonly ILog log;
+        private readonly ICodexNodeHooks hooks;
+        private readonly TransferSpeeds transferSpeeds;
+        private string peerId = string.Empty;
+        private string nodeId = string.Empty;
+        private readonly CodexAccess codexAccess;
+        private readonly IFileManager fileManager;
+
+        public CodexNode(ILog log, CodexAccess codexAccess, IFileManager fileManager, IMarketplaceAccess marketplaceAccess, ICodexNodeHooks hooks)
+        {
+            this.codexAccess = codexAccess;
+            this.fileManager = fileManager;
+            Marketplace = marketplaceAccess;
+            this.hooks = hooks;
+            Version = new DebugInfoVersion();
+            transferSpeeds = new TransferSpeeds();
+
+            this.log = new LogPrefixer(log, $"{GetName()} ");
+        }
+
+        public void Awake()
+        {
+            hooks.OnNodeStarting(codexAccess.GetStartUtc(), codexAccess.GetImageName(), codexAccess.GetEthAccount());
+        }
+
+        public void Initialize()
+        {
+            InitializePeerNodeId();
+            InitializeLogReplacements();
+
+            hooks.OnNodeStarted(this, peerId, nodeId);
+        }
+
+        public IMarketplaceAccess Marketplace { get; }
+        public DebugInfoVersion Version { get; private set; }
+        public ITransferSpeeds TransferSpeeds { get => transferSpeeds; }
+
+        public StoragePurchase? GetPurchaseStatus(string purchaseId)
+        {
+            return codexAccess.GetPurchaseStatus(purchaseId);
+        }
+
+        public EthAddress EthAddress 
+        {
+            get
+            {
+                EnsureMarketplace();
+                return codexAccess.GetEthAccount()!.EthAddress;
+            }
+        }
+
+        public EthAccount EthAccount
+        {
+            get
+            {
+                EnsureMarketplace();
+                return codexAccess.GetEthAccount()!;
+            }
+        }
+
+        public string GetName()
+        {
+            return codexAccess.GetName();
+        }
+
+        public string GetImageName()
+        {
+            return codexAccess.GetImageName();
+        }
+
+        public string GetPeerId()
+        {
+            return peerId;
+        }
+
+        public DebugInfo GetDebugInfo(bool log = false)
+        {
+            var debugInfo = codexAccess.GetDebugInfo();
+            if (log)
+            {
+                var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
+                Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
+            }
+            return debugInfo;
+        }
+
+        public void SetLogLevel(string logLevel)
+        {
+            codexAccess.SetLogLevel(logLevel);
+        }
+
+        public string GetSpr()
+        {
+            return codexAccess.GetSpr();
+        }
+
+        public DebugPeer GetDebugPeer(string peerId)
+        {
+            return codexAccess.GetDebugPeer(peerId);
+        }
+
+        public ContentId UploadFile(TrackedFile file)
+        {
+            return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"");
+        }
+
+        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition)
+        {
+            using var fileStream = File.OpenRead(file.Filename);
+            var uniqueId = Guid.NewGuid().ToString();
+            var size = file.GetFilesize();
+
+            hooks.OnFileUploading(uniqueId, size);
+
+            var input = new UploadInput(contentType, contentDisposition, fileStream);
+            var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
+            var measurement = Stopwatch.Measure(log, logMessage, () =>
+            {
+                return codexAccess.UploadFile(input);
+            });
+
+            var response = measurement.Value;
+            transferSpeeds.AddUploadSample(size, measurement.Duration);
+
+            if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
+            if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
+
+            Log($"Uploaded file {file.Describe()}. Received contentId: '{response}'.");
+
+            var cid = new ContentId(response);
+            hooks.OnFileUploaded(uniqueId, size, cid);
+            return cid;
+        }
+
+        public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
+        {
+            var file = fileManager.CreateEmptyFile(fileLabel);
+            hooks.OnFileDownloading(contentId);
+            Log($"Downloading '{contentId}'...");
+
+            var logMessage = $"Downloaded '{contentId}' to '{file.Filename}'";
+            var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file));
+
+            var size = file.GetFilesize();
+            transferSpeeds.AddDownloadSample(size, measurement);
+            hooks.OnFileDownloaded(size, contentId);
+
+            return file;
+        }
+
+        public LocalDataset DownloadStreamless(ContentId cid)
+        {
+            Log($"Downloading streamless '{cid}' (no-wait)");
+            return codexAccess.DownloadStreamless(cid);
+        }
+
+        public LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size)
+        {
+            Log($"Downloading streamless '{cid}' (wait till finished)");
+
+            var sw = Stopwatch.Measure(log, nameof(DownloadStreamlessWait), () =>
+            {
+                var startSpace = Space();
+                var result = codexAccess.DownloadStreamless(cid);
+                WaitUntilQuotaUsedIncreased(startSpace, size);
+                return result;
+            });
+
+            return sw.Value;
+        }
+
+        public LocalDataset DownloadManifestOnly(ContentId cid)
+        {
+            Log($"Downloading manifest-only '{cid}'");
+            return codexAccess.DownloadManifestOnly(cid);
+        }
+
+        public LocalDatasetList LocalFiles()
+        {
+            return codexAccess.LocalFiles();
+        }
+
+        public CodexSpace Space()
+        {
+            return codexAccess.Space();
+        }
+
+        public void ConnectToPeer(ICodexNode node)
+        {
+            var peer = (CodexNode)node;
+
+            Log($"Connecting to peer {peer.GetName()}...");
+            var peerInfo = node.GetDebugInfo();
+            codexAccess.ConnectToPeer(peerInfo.Id, GetPeerMultiAddresses(peer, peerInfo));
+
+            Log($"Successfully connected to peer {peer.GetName()}.");
+        }
+
+        public void DeleteDataDirFolder()
+        {
+            codexAccess.DeleteDataDirFolder();
+        }
+
+        public void Stop(bool waitTillStopped)
+        {
+            Log("Stopping...");
+            hooks.OnNodeStopping();
+            codexAccess.Stop(waitTillStopped);
+        }
+
+        public IDownloadedLog DownloadLog(string additionalName = "")
+        {
+            return codexAccess.DownloadLog(additionalName);
+        }
+
+        public Address GetDiscoveryEndpoint()
+        {
+            return codexAccess.GetDiscoveryEndpoint();
+        }
+
+        public Address GetApiEndpoint()
+        {
+            return codexAccess.GetApiEndpoint();
+        }
+
+        public Address GetListenEndpoint()
+        {
+            return codexAccess.GetListenEndpoint();
+        }
+
+        public Address GetMetricsScrapeTarget()
+        {
+            var address = codexAccess.GetMetricsEndpoint();
+            if (address == null) throw new Exception("Metrics ScrapeTarget accessed, but node was not started with EnableMetrics()");
+            return address;
+        }
+
+        public bool HasCrashed()
+        {
+            return codexAccess.HasCrashed();
+        }
+
+        public override string ToString()
+        {
+            return $"CodexNode:{GetName()}";
+        }
+
+        private void InitializePeerNodeId()
+        {
+            var debugInfo = Time.Retry(codexAccess.GetDebugInfo, "ensure online");
+            if (!debugInfo.Version.IsValid())
+            {
+                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
+            }
+
+            peerId = debugInfo.Id;
+            nodeId = debugInfo.Table.LocalNode.NodeId;
+            Version = debugInfo.Version;
+        }
+
+        private void InitializeLogReplacements()
+        {
+            var nodeName = GetName();
+
+            log.AddStringReplace(peerId, nodeName);
+            log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName);
+            log.AddStringReplace(nodeId, nodeName);
+            log.AddStringReplace(CodexUtils.ToShortId(nodeId), nodeName);
+        }
+
+        private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)
+        {
+            var peerId = peer.GetDiscoveryEndpoint().Host
+                .Replace("http://", "")
+                .Replace("https://", "");
+
+            return peerInfo.Addrs.Select(a => a
+                .Replace("0.0.0.0", peerId))
+                .ToArray();
+        }
+
+        private void DownloadToFile(string contentId, TrackedFile file)
+        {
+            using var fileStream = File.OpenWrite(file.Filename);
+            var timeout = TimeSpan.FromMinutes(2.0); // todo: make this user-controllable.
+            try
+            {
+                // Type of stream generated by openAPI client does not support timeouts.
+                var start = DateTime.UtcNow;
+                var cts = new CancellationTokenSource();
+                var downloadTask = Task.Run(() =>
+                {
+                    using var downloadStream = codexAccess.DownloadFile(contentId);
+                    downloadStream.CopyTo(fileStream);
+                }, cts.Token);
+                
+                while (DateTime.UtcNow - start < timeout)
+                {
+                    if (downloadTask.IsFaulted) throw downloadTask.Exception;
+                    if (downloadTask.IsCompletedSuccessfully) return;
+                    Thread.Sleep(100);
+                }
+
+                cts.Cancel();
+                throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
+            }
+            catch (Exception ex)
+            {
+                Log($"Failed to download file '{contentId}': {ex}");
+                throw;
+            }
+        }
+
+        public void WaitUntilQuotaUsedIncreased(CodexSpace startSpace, ByteSize expectedIncreaseOfQuotaUsed)
+        {
+            WaitUntilQuotaUsedIncreased(startSpace, expectedIncreaseOfQuotaUsed, TimeSpan.FromMinutes(2));
+        }
+
+        public void WaitUntilQuotaUsedIncreased(
+            CodexSpace startSpace,
+            ByteSize expectedIncreaseOfQuotaUsed,
+            TimeSpan maxTimeout)
+        {
+            Log($"Waiting until quotaUsed " +
+                $"(start: {startSpace.QuotaUsedBytes}) " +
+                $"increases by {expectedIncreaseOfQuotaUsed} " +
+                $"to reach {startSpace.QuotaUsedBytes + expectedIncreaseOfQuotaUsed.SizeInBytes}");
+
+            var retry = new Retry($"Checking local space for quotaUsed increase of {expectedIncreaseOfQuotaUsed}",
+            maxTimeout: maxTimeout,
+            sleepAfterFail: TimeSpan.FromSeconds(3),
+            onFail: f => { });
+
+            retry.Run(() =>
+            {
+                var space = Space();
+                var increase = space.QuotaUsedBytes - startSpace.QuotaUsedBytes;
+
+                if (increase < expectedIncreaseOfQuotaUsed.SizeInBytes)
+                    throw new Exception($"Expected quota-used not reached. " +
+                        $"Expected increase: {expectedIncreaseOfQuotaUsed.SizeInBytes} " +
+                        $"Actual increase: {increase} " +
+                        $"Actual used: {space.QuotaUsedBytes}");
+            });
+        }
+
+        private void EnsureMarketplace()
+        {
+            if (codexAccess.GetEthAccount() == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+    }
+}

ProjectPlugins/CodexClient/CodexNodeFactory.cs

@@ -0,0 +1,52 @@
+using CodexClient.Hooks;
+using FileUtils;
+using Logging;
+using WebUtils;
+
+namespace CodexClient
+{
+    public class CodexNodeFactory
+    {
+        private readonly ILog log;
+        private readonly IFileManager fileManager;
+        private readonly CodexHooksFactory hooksFactory;
+        private readonly IHttpFactory httpFactory;
+        private readonly IProcessControlFactory processControlFactory;
+
+        public CodexNodeFactory(ILog log, IFileManager fileManager, CodexHooksFactory hooksFactory, IHttpFactory httpFactory, IProcessControlFactory processControlFactory)
+        {
+            this.log = log;
+            this.fileManager = fileManager;
+            this.hooksFactory = hooksFactory;
+            this.httpFactory = httpFactory;
+            this.processControlFactory = processControlFactory;
+        }
+
+        public CodexNodeFactory(ILog log, HttpFactory httpFactory, string dataDir)
+            : this(log, new FileManager(log, dataDir), new CodexHooksFactory(), httpFactory, new DoNothingProcessControlFactory())
+        {
+        }
+
+        public CodexNodeFactory(ILog log, string dataDir)
+            : this(log, new HttpFactory(log), dataDir)
+        {
+        }
+
+        public ICodexNode CreateCodexNode(ICodexInstance instance)
+        {
+            var processControl = processControlFactory.CreateProcessControl(instance);
+            var access = new CodexAccess(log, httpFactory, processControl, instance);
+            var hooks = hooksFactory.CreateHooks(access.GetName());
+            var marketplaceAccess = CreateMarketplaceAccess(instance, access, hooks);
+            var node =  new CodexNode(log, access, fileManager, marketplaceAccess, hooks);
+            node.Initialize();
+            return node;
+        }
+
+        private IMarketplaceAccess CreateMarketplaceAccess(ICodexInstance instance, CodexAccess access, ICodexNodeHooks hooks)
+        {
+            if (instance.EthAccount == null) return new MarketplaceUnavailable();
+            return new MarketplaceAccess(log, access, hooks);
+        }
+    }
+}

ProjectPlugins/CodexClient/CodexTypes.cs

@@ -1,7 +1,7 @@
 using Newtonsoft.Json;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class DebugInfo
     {
@@ -109,6 +109,16 @@ namespace CodexPlugin
         {
             return HashCode.Combine(Id);
         }
+
+        public static bool operator ==(ContentId a, ContentId b)
+        {
+            return a.Id == b.Id;
+        }
+
+        public static bool operator !=(ContentId a, ContentId b)
+        {
+            return a.Id != b.Id;
+        }
     }
 
     public class CodexSpace

ProjectPlugins/CodexClient/CodexUtils.cs

@@ -1,4 +1,4 @@
-namespace CodexPlugin
+namespace CodexClient
 {
     public static class CodexUtils
     {

ProjectPlugins/CodexClient/Hooks/CodexHooksFactory.cs

@@ -1,7 +1,6 @@
-using GethPlugin;
-using Utils;
+using Utils;
 
-namespace CodexPlugin.Hooks
+namespace CodexClient.Hooks
 {
     public interface ICodexHooksProvider
     {
@@ -10,11 +9,14 @@ namespace CodexPlugin.Hooks
 
     public class CodexHooksFactory
     {
-        public ICodexHooksProvider Provider { get; set; } = new DoNothingHooksProvider();
+        public List<ICodexHooksProvider> Providers { get; } = new List<ICodexHooksProvider>();
 
         public ICodexNodeHooks CreateHooks(string nodeName)
         {
-            return Provider.CreateHooks(nodeName);
+            if (Providers.Count == 0) return new DoNothingCodexHooks();
+
+            var hooks = Providers.Select(p => p.CreateHooks(nodeName)).ToArray();
+            return new MuxingCodexNodeHooks(hooks);
         }
     }
 
@@ -44,7 +46,7 @@ namespace CodexPlugin.Hooks
         {
         }
 
-        public void OnNodeStarted(string peerId, string nodeId)
+        public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
         {
         }
 

ProjectPlugins/CodexClient/Hooks/CodexNodeHooks.cs

@@ -0,0 +1,78 @@
+using Utils;
+
+namespace CodexClient.Hooks
+{
+    public interface ICodexNodeHooks
+    {
+        void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount);
+        void OnNodeStarted(ICodexNode node, string peerId, string nodeId);
+        void OnNodeStopping();
+        void OnFileUploading(string uid, ByteSize size);
+        void OnFileUploaded(string uid, ByteSize size, ContentId cid);
+        void OnFileDownloading(ContentId cid);
+        void OnFileDownloaded(ByteSize size, ContentId cid);
+        void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract);
+        void OnStorageContractUpdated(StoragePurchase purchaseStatus);
+        void OnStorageAvailabilityCreated(StorageAvailability response);
+    }
+
+    public class MuxingCodexNodeHooks : ICodexNodeHooks
+    {
+        private readonly ICodexNodeHooks[] backingHooks;
+
+        public MuxingCodexNodeHooks(ICodexNodeHooks[] backingHooks)
+        {
+            this.backingHooks = backingHooks;
+        }
+
+        public void OnFileDownloaded(ByteSize size, ContentId cid)
+        {
+            foreach (var h in backingHooks) h.OnFileDownloaded(size, cid);
+        }
+
+        public void OnFileDownloading(ContentId cid)
+        {
+            foreach (var h in backingHooks) h.OnFileDownloading(cid);
+        }
+
+        public void OnFileUploaded(string uid, ByteSize size, ContentId cid)
+        {
+            foreach (var h in backingHooks) h.OnFileUploaded(uid, size, cid);
+        }
+
+        public void OnFileUploading(string uid, ByteSize size)
+        {
+            foreach (var h in backingHooks) h.OnFileUploading(uid, size);
+        }
+
+        public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
+        {
+            foreach (var h in backingHooks) h.OnNodeStarted(node, peerId, nodeId);
+        }
+
+        public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount)
+        {
+            foreach (var h in backingHooks) h.OnNodeStarting(startUtc, image, ethAccount);
+        }
+
+        public void OnNodeStopping()
+        {
+            foreach (var h in backingHooks) h.OnNodeStopping();
+        }
+
+        public void OnStorageAvailabilityCreated(StorageAvailability response)
+        {
+            foreach (var h in backingHooks) h.OnStorageAvailabilityCreated(response);
+        }
+
+        public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract)
+        {
+            foreach (var h in backingHooks) h.OnStorageContractSubmitted(storagePurchaseContract);
+        }
+
+        public void OnStorageContractUpdated(StoragePurchase purchaseStatus)
+        {
+            foreach (var h in backingHooks) h.OnStorageContractUpdated(purchaseStatus);
+        }
+    }
+}

ProjectPlugins/CodexClient/Mapper.cs

@@ -1,10 +1,9 @@
-using CodexContractsPlugin;
-using CodexOpenApi;
+using CodexOpenApi;
 using Newtonsoft.Json.Linq;
 using System.Numerics;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class Mapper
     {
@@ -15,20 +14,12 @@ namespace CodexPlugin
                 Id = debugInfo.Id,
                 Spr = debugInfo.Spr,
                 Addrs = debugInfo.Addrs.ToArray(),
-                AnnounceAddresses = JArray(debugInfo.AdditionalProperties, "announceAddresses").Select(x => x.ToString()).ToArray(),
+                AnnounceAddresses = debugInfo.AnnounceAddresses.ToArray(),
                 Version = Map(debugInfo.Codex),
                 Table = Map(debugInfo.Table)
             };
         }
 
-        public LocalDatasetList Map(LocalDatasetListJson json)
-        {
-            return new LocalDatasetList
-            {
-                Content = json.Content.Select(Map).ToArray()
-            };
-        }
-
         public LocalDatasetList Map(CodexOpenApi.DataList dataList)
         {
             return new LocalDatasetList
@@ -46,22 +37,13 @@ namespace CodexPlugin
             };
         }
 
-        public LocalDataset Map(LocalDatasetListJsonItem item)
-        {
-            return new LocalDataset
-            {
-                Cid = new ContentId(item.Cid),
-                Manifest = MapManifest(item.Manifest)
-            };
-        }
-
         public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
         {
             return new CodexOpenApi.SalesAvailabilityCREATE
             {
                 Duration = ToDecInt(availability.MaxDuration.TotalSeconds),
-                MinPrice = ToDecInt(availability.MinPriceForTotalSpace),
-                MaxCollateral = ToDecInt(availability.MaxCollateral),
+                MinPricePerBytePerSecond = ToDecInt(availability.MinPricePerBytePerSecond),
+                TotalCollateral = ToDecInt(availability.TotalCollateral),
                 TotalSize = ToDecInt(availability.TotalSpace.SizeInBytes)
             };
         }
@@ -72,27 +54,27 @@ namespace CodexPlugin
             {
                 Duration = ToDecInt(purchase.Duration.TotalSeconds),
                 ProofProbability = ToDecInt(purchase.ProofProbability),
-                Reward = ToDecInt(purchase.PricePerSlotPerSecond),
-                Collateral = ToDecInt(purchase.RequiredCollateral),
+                PricePerBytePerSecond = ToDecInt(purchase.PricePerBytePerSecond),
+                CollateralPerByte = ToDecInt(purchase.CollateralPerByte),
                 Expiry = ToDecInt(purchase.Expiry.TotalSeconds),
                 Nodes = Convert.ToInt32(purchase.MinRequiredNumberOfNodes),
                 Tolerance = Convert.ToInt32(purchase.NodeFailureTolerance)
             };
         }
 
-        public StorageAvailability[] Map(ICollection<SalesAvailabilityREAD> availabilities)
+        public StorageAvailability[] Map(ICollection<CodexOpenApi.SalesAvailabilityREAD> availabilities)
         {
             return availabilities.Select(a => Map(a)).ToArray();
         }
 
-        public StorageAvailability Map(SalesAvailabilityREAD availability)
+        public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD availability)
         {
             return new StorageAvailability
             (
                 ToByteSize(availability.TotalSize),
                 ToTimespan(availability.Duration),
-                new TestToken(ToBigIng(availability.MinPrice)),
-                new TestToken(ToBigIng(availability.MaxCollateral))
+                new TestToken(ToBigIng(availability.MinPricePerBytePerSecond)),
+                new TestToken(ToBigIng(availability.TotalCollateral))
             )
             {
                 Id = availability.Id,
@@ -100,49 +82,80 @@ namespace CodexPlugin
             };
         }
 
-        // TODO: Fix openapi spec for this call.
-        //public StoragePurchase Map(CodexOpenApi.Purchase purchase)
-        //{
-        //    return new StoragePurchase(Map(purchase.Request))
-        //    {
-        //        State = purchase.State,
-        //        Error = purchase.Error
-        //    };
-        //}
+        public StoragePurchase Map(CodexOpenApi.Purchase purchase)
+        {
+            return new StoragePurchase
+            {
+                Request = Map(purchase.Request),
+                State = purchase.State.ToString(), //Map(purchase.State),
+                Error = purchase.Error
+            };
+        }
 
-        //public StorageRequest Map(CodexOpenApi.StorageRequest request)
-        //{
-        //    return new StorageRequest(Map(request.Ask), Map(request.Content))
-        //    {
-        //        Id = request.Id,
-        //        Client = request.Client,
-        //        Expiry = TimeSpan.FromSeconds(Convert.ToInt64(request.Expiry)),
-        //        Nonce = request.Nonce
-        //    };
-        //}
+        public StoragePurchaseState Map(PurchaseState purchaseState)
+        {
+            // TODO: to be re-enabled when marketplace api lines up with openapi.yaml.
 
-        //public StorageAsk Map(CodexOpenApi.StorageAsk ask)
-        //{
-        //    return new StorageAsk
-        //    {
-        //        Duration = TimeSpan.FromSeconds(Convert.ToInt64(ask.Duration)),
-        //        MaxSlotLoss = ask.MaxSlotLoss,
-        //        ProofProbability = ask.ProofProbability,
-        //        Reward = Convert.ToDecimal(ask.Reward).TstWei(),
-        //        Slots = ask.Slots,
-        //        SlotSize = new ByteSize(Convert.ToInt64(ask.SlotSize))
-        //    };
-        //}
+            // Explicit mapping: If the API changes, we will get compile errors here.
+            // That's what we want.
+            switch (purchaseState)
+            {
+                case PurchaseState.Cancelled:
+                    return StoragePurchaseState.Cancelled;
+                case PurchaseState.Error:
+                    return StoragePurchaseState.Error;
+                case PurchaseState.Failed:
+                    return StoragePurchaseState.Failed;
+                case PurchaseState.Finished:
+                    return StoragePurchaseState.Finished;
+                case PurchaseState.Pending:
+                    return StoragePurchaseState.Pending;
+                case PurchaseState.Started:
+                    return StoragePurchaseState.Started;
+                case PurchaseState.Submitted:
+                    return StoragePurchaseState.Submitted;
+                case PurchaseState.Unknown:
+                    return StoragePurchaseState.Unknown;
+            }
 
-        //public StorageContent Map(CodexOpenApi.Content content)
-        //{
-        //    return new StorageContent
-        //    {
-        //        Cid = content.Cid
-        //    };
-        //}
+            throw new Exception("API incompatibility detected. Unknown purchaseState: " + purchaseState.ToString());
+        }
 
-        public CodexSpace Map(Space space)
+        public StorageRequest Map(CodexOpenApi.StorageRequest request)
+        {
+            return new StorageRequest
+            {
+                Ask = Map(request.Ask),
+                Content = Map(request.Content),
+                Id = request.Id,
+                Client = request.Client,
+                Expiry = request.Expiry,
+                Nonce = request.Nonce
+            };
+        }
+
+        public StorageAsk Map(CodexOpenApi.StorageAsk ask)
+        {
+            return new StorageAsk
+            {
+                Duration = ask.Duration,
+                MaxSlotLoss = ask.MaxSlotLoss,
+                ProofProbability = ask.ProofProbability,
+                PricePerBytePerSecond = ask.PricePerBytePerSecond,
+                Slots = ask.Slots,
+                SlotSize = ask.SlotSize
+            };
+        }
+
+        public StorageContent Map(CodexOpenApi.Content content)
+        {
+            return new StorageContent
+            {
+                Cid = content.Cid
+            };
+        }
+
+        public CodexSpace Map(CodexOpenApi.Space space)
         {
             return new CodexSpace
             {
@@ -153,7 +166,7 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoVersion Map(CodexVersion obj)
+        private DebugInfoVersion Map(CodexOpenApi.CodexVersion obj)
         {
             return new DebugInfoVersion
             {
@@ -162,7 +175,7 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoTable Map(PeersTable obj)
+        private DebugInfoTable Map(CodexOpenApi.PeersTable obj)
         {
             return new DebugInfoTable
             {
@@ -171,7 +184,7 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoTableNode Map(Node? token)
+        private DebugInfoTableNode Map(CodexOpenApi.Node? token)
         {
             if (token == null) return new DebugInfoTableNode();
             return new DebugInfoTableNode
@@ -184,7 +197,7 @@ namespace CodexPlugin
             };
         }
 
-        private DebugInfoTableNode[] Map(ICollection<Node> nodes)
+        private DebugInfoTableNode[] Map(ICollection<CodexOpenApi.Node> nodes)
         {
             if (nodes == null || nodes.Count == 0)
             {
@@ -205,18 +218,6 @@ namespace CodexPlugin
             };
         }
 
-        public Manifest MapManifest(LocalDatasetListJsonItemManifest manifest)
-        {
-            return new Manifest
-            {
-                // needs update
-                BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
-                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.DatasetSize)),
-                RootHash = manifest.TreeCid,
-                Protected = manifest.Protected
-            };
-        }
-
         private JArray JArray(IDictionary<string, object> map, string name)
         {
             return (JArray)map[name];
@@ -272,42 +273,4 @@ namespace CodexPlugin
             return new ByteSize(Convert.ToInt64(size));
         }
     }
-
-
-//"content": [
-//        {
-//            "cid": "zDvZRwzkxLxVaGces3kpkHjo8EcTPXudvYMfNxdoH21Ask1Js5fJ",
-//            "manifest": {
-//                "treeCid": "zDzSvJTf8GBRyEDNuAzXS9VnRfh8cNuYuRPwTLW6RUQReSgKnhCt",
-//                "datasetSize": 5242880,
-//                "blockSize": 65536,
-//                "filename": null,
-//                "mimetype": "application/octet-stream",
-//                "uploadedAt": 1731426230,
-//                "protected": false
-//            }
-//        }
-//    ]
-
-    public class LocalDatasetListJson
-    {
-        public LocalDatasetListJsonItem[] Content { get; set; } = Array.Empty<LocalDatasetListJsonItem>();
-    }
-
-    public class LocalDatasetListJsonItem
-    {
-        public string Cid { get; set; } = string.Empty;
-        public LocalDatasetListJsonItemManifest Manifest { get; set; } = new();
-    }
-
-    public class LocalDatasetListJsonItemManifest
-    {
-        public string TreeCid { get; set; } = string.Empty;
-        public int DatasetSize { get; set; }
-        public int BlockSize { get; set; }
-        public string? Filename { get; set; } = string.Empty;
-        public string? MimeType { get; set; } = string.Empty;
-        public int? UploadedAt { get; set; } 
-        public bool Protected { get; set; }
-    }
 }

ProjectPlugins/CodexClient/MarketplaceAccess.cs

@@ -1,8 +1,8 @@
-using CodexPlugin.Hooks;
+using CodexClient.Hooks;
 using Logging;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public interface IMarketplaceAccess
     {
@@ -27,8 +27,12 @@ namespace CodexPlugin
         public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
         {
             purchase.Log(log);
+            var swResult = Stopwatch.Measure(log, nameof(RequestStorage), () =>
+            {
+                return codexAccess.RequestStorage(purchase);
+            });
 
-            var response = codexAccess.RequestStorage(purchase);
+            var response = swResult.Value;
 
             if (string.IsNullOrEmpty(response) ||
                 response == "Unable to encode manifest" ||
@@ -42,12 +46,7 @@ namespace CodexPlugin
 
             Log($"Storage requested successfully. PurchaseId: '{response}'.");
 
-            var contract = new StoragePurchaseContract(log, codexAccess, response, purchase, hooks);
-            contract.WaitForStorageContractSubmitted();
-
-            hooks.OnStorageContractSubmitted(contract);
-
-            return contract;
+            return new StoragePurchaseContract(log, codexAccess, response, purchase, hooks);
         }
 
         public string MakeStorageAvailable(StorageAvailability availability)
@@ -72,7 +71,7 @@ namespace CodexPlugin
 
         private void Log(string msg)
         {
-            log.Log($"{codexAccess.Container.Containers.Single().Name} {msg}");
+            log.Log($"{codexAccess.GetName()} {msg}");
         }
     }
 

ProjectPlugins/CodexClient/MarketplaceTypes.cs

@@ -1,10 +1,7 @@
-using CodexContractsPlugin;
-using CodexOpenApi;
-using Logging;
-using System.Data;
+using Logging;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public class StoragePurchaseRequest
     {
@@ -14,8 +11,8 @@ namespace CodexPlugin
         }
 
         public ContentId ContentId { get; set; }
-        public TestToken PricePerSlotPerSecond { get; set; } = 1.TstWei();
-        public TestToken RequiredCollateral { get; set; } = 1.TstWei();
+        public TestToken PricePerBytePerSecond { get; set; } = 1.TstWei();
+        public TestToken CollateralPerByte { get; set; } = 1.TstWei();
         public uint MinRequiredNumberOfNodes { get; set; }
         public uint NodeFailureTolerance { get; set; }
         public int ProofProbability { get; set; }
@@ -25,8 +22,8 @@ namespace CodexPlugin
         public void Log(ILog log)
         {
             log.Log($"Requesting storage for: {ContentId.Id}... (" +
-                $"pricePerSlotPerSecond: {PricePerSlotPerSecond}, " +
-                $"requiredCollateral: {RequiredCollateral}, " +
+                $"pricePerBytePerSecond: {PricePerBytePerSecond}, " +
+                $"collateralPerByte: {CollateralPerByte}, " +
                 $"minRequiredNumberOfNodes: {MinRequiredNumberOfNodes}, " +
                 $"nodeFailureTolerance: {NodeFailureTolerance}, " +
                 $"proofProbability: {ProofProbability}, " +
@@ -48,6 +45,18 @@ namespace CodexPlugin
         public bool IsSubmitted => State.ToLowerInvariant().Contains("submitted");
     }
 
+    public enum StoragePurchaseState
+    {
+        Cancelled = 0,
+        Error = 1,
+        Failed = 2,
+        Finished = 3,
+        Pending = 4,
+        Started = 5,
+        Submitted = 6,
+        Unknown = 7,
+    }
+
     public class StorageRequest
     {
         public string Id { get; set; } = string.Empty;
@@ -64,7 +73,7 @@ namespace CodexPlugin
         public string SlotSize { get; set; } = string.Empty;
         public string Duration { get; set; } = string.Empty;
         public string ProofProbability { get; set; } = string.Empty;
-        public string Reward { get; set; } = string.Empty;
+        public string PricePerBytePerSecond { get; set; } = string.Empty;
         public int MaxSlotLoss { get; set; }
     }
 
@@ -77,19 +86,19 @@ namespace CodexPlugin
 
     public class StorageAvailability
     {
-        public StorageAvailability(ByteSize totalSpace, TimeSpan maxDuration, TestToken minPriceForTotalSpace, TestToken maxCollateral)
+        public StorageAvailability(ByteSize totalSpace, TimeSpan maxDuration, TestToken minPricePerBytePerSecond, TestToken totalCollateral)
         {
             TotalSpace = totalSpace;
             MaxDuration = maxDuration;
-            MinPriceForTotalSpace = minPriceForTotalSpace;
-            MaxCollateral = maxCollateral;
+            MinPricePerBytePerSecond = minPricePerBytePerSecond;
+            TotalCollateral = totalCollateral;
         }
 
         public string Id { get; set; } = string.Empty;
         public ByteSize TotalSpace { get; }
         public TimeSpan MaxDuration { get; }
-        public TestToken MinPriceForTotalSpace { get; }
-        public TestToken MaxCollateral { get; } 
+        public TestToken MinPricePerBytePerSecond { get; }
+        public TestToken TotalCollateral { get; } 
         public ByteSize FreeSpace { get; set; } = ByteSize.Zero;
 
         public void Log(ILog log)
@@ -97,8 +106,8 @@ namespace CodexPlugin
             log.Log($"Storage Availability: (" +
                 $"totalSize: {TotalSpace}, " +
                 $"maxDuration: {Time.FormatDuration(MaxDuration)}, " + 
-                $"minPriceForTotalSpace: {MinPriceForTotalSpace}, " +
-                $"maxCollateral: {MaxCollateral})");
+                $"minPricePerBytePerSecond: {MinPricePerBytePerSecond}, " +
+                $"totalCollateral: {TotalCollateral})");
         }
     }
 }

ProjectPlugins/CodexClient/ProcessControl.cs

@@ -0,0 +1,46 @@
+using Logging;
+
+namespace CodexClient
+{
+    public interface IProcessControlFactory
+    {
+        IProcessControl CreateProcessControl(ICodexInstance instance);
+    }
+
+    public interface IProcessControl
+    {
+        void Stop(bool waitTillStopped);
+        IDownloadedLog DownloadLog(LogFile file);
+        void DeleteDataDirFolder();
+        bool HasCrashed();
+    }
+
+    public class DoNothingProcessControlFactory : IProcessControlFactory
+    {
+        public IProcessControl CreateProcessControl(ICodexInstance instance)
+        {
+            return new DoNothingProcessControl();
+        }
+    }
+
+    public class DoNothingProcessControl : IProcessControl
+    {
+        public void DeleteDataDirFolder()
+        {
+        }
+
+        public IDownloadedLog DownloadLog(LogFile file)
+        {
+            throw new NotImplementedException("Not supported by DoNothingProcessControl");
+        }
+
+        public bool HasCrashed()
+        {
+            return false;
+        }
+
+        public void Stop(bool waitTillStopped)
+        {
+        }
+    }
+}

ProjectPlugins/CodexClient/StoragePurchaseContract.cs

@@ -1,18 +1,20 @@
-using CodexPlugin.Hooks;
+using CodexClient.Hooks;
 using Logging;
 using Newtonsoft.Json;
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public interface IStoragePurchaseContract
     {
         string PurchaseId { get; }
         StoragePurchaseRequest Purchase { get; }
         ContentId ContentId { get; }
+        StoragePurchase? GetStatus();
         void WaitForStorageContractSubmitted();
         void WaitForStorageContractStarted();
         void WaitForStorageContractFinished();
+        void WaitForContractFailed();
     }
 
     public class StoragePurchaseContract : IStoragePurchaseContract
@@ -20,11 +22,13 @@ namespace CodexPlugin
         private readonly ILog log;
         private readonly CodexAccess codexAccess;
         private readonly ICodexNodeHooks hooks;
-        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30);
+        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(60);
         private readonly DateTime contractPendingUtc = DateTime.UtcNow;
         private DateTime? contractSubmittedUtc = DateTime.UtcNow;
         private DateTime? contractStartedUtc;
         private DateTime? contractFinishedUtc;
+        private string lastState = string.Empty;
+        private ContentId encodedContentId = new ContentId();
 
         public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase, ICodexNodeHooks hooks)
         {
@@ -33,21 +37,40 @@ namespace CodexPlugin
             PurchaseId = purchaseId;
             Purchase = purchase;
             this.hooks = hooks;
-            ContentId = new ContentId(codexAccess.GetPurchaseStatus(purchaseId).Request.Content.Cid);
         }
 
         public string PurchaseId { get; }
         public StoragePurchaseRequest Purchase { get; }
-        public ContentId ContentId { get; }
+        public ContentId ContentId
+        {
+            get
+            {
+                if (string.IsNullOrEmpty(encodedContentId.Id)) GetStatus();
+                return encodedContentId;
+            }
+        }
 
         public TimeSpan? PendingToSubmitted => contractSubmittedUtc - contractPendingUtc;
         public TimeSpan? SubmittedToStarted => contractStartedUtc - contractSubmittedUtc;
         public TimeSpan? SubmittedToFinished => contractFinishedUtc - contractSubmittedUtc;
 
+        public StoragePurchase? GetStatus()
+        {
+            var status = codexAccess.GetPurchaseStatus(PurchaseId);
+            if (status != null)
+            {
+                encodedContentId = new ContentId(status.Request.Content.Cid);
+            }
+            return status;
+        }
+
         public void WaitForStorageContractSubmitted()
         {
-            WaitForStorageContractState(gracePeriod, "submitted", sleep: 200);
+            var timeout = Purchase.Expiry + gracePeriod;
+            var raiseHook = lastState != "submitted";
+            WaitForStorageContractState(timeout, "submitted", sleep: 200);
             contractSubmittedUtc = DateTime.UtcNow;
+            if (raiseHook) hooks.OnStorageContractSubmitted(this);
             LogSubmittedDuration();
             AssertDuration(PendingToSubmitted, gracePeriod, nameof(PendingToSubmitted));
         }
@@ -76,14 +99,19 @@ namespace CodexPlugin
             AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished));
         }
 
-        public StoragePurchase GetPurchaseStatus(string purchaseId)
+        public void WaitForContractFailed()
         {
-            return codexAccess.GetPurchaseStatus(purchaseId);
+            if (!contractStartedUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value;
+            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
+            WaitForStorageContractState(timeout, "failed");
         }
 
         private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000)
         {
-            var lastState = "";
             var waitStart = DateTime.UtcNow;
 
             Log($"Waiting for {Time.FormatDuration(timeout)} to reach state '{desiredState}'.");
@@ -115,7 +143,7 @@ namespace CodexPlugin
         private void LogSubmittedDuration()
         {
             Log($"Pending to Submitted in {Time.FormatDuration(PendingToSubmitted)} " +
-                $"( < {Time.FormatDuration(gracePeriod)})");
+                $"( < {Time.FormatDuration(Purchase.Expiry + gracePeriod)})");
         }
 
         private void LogStartedDuration()

ProjectPlugins/CodexClient/TransferSpeeds.cs

@@ -1,6 +1,6 @@
 using Utils;
 
-namespace CodexPlugin
+namespace CodexClient
 {
     public interface ITransferSpeeds
     {

ProjectPlugins/CodexClient/openapi.yaml

@@ -6,7 +6,7 @@ info:
   description: "List of endpoints and interfaces available to Codex API users"
 
 security:
-  - { }
+  - {}
 
 components:
   schemas:
@@ -50,9 +50,9 @@ components:
       type: string
       description: Address of Ethereum address
 
-    Reward:
+    PricePerBytePerSecond:
       type: string
-      description: The maximum amount of tokens paid per second per slot to hosts the client is willing to pay
+      description: The amount of tokens paid per byte per second per slot to hosts the client is willing to pay
 
     Duration:
       type: string
@@ -138,6 +138,10 @@ components:
           description: Path of the data repository where all nodes data are stored
         spr:
           $ref: "#/components/schemas/SPR"
+        announceAddresses:
+          type: array
+          items:
+            $ref: "#/components/schemas/MultiAddress"
         table:
           $ref: "#/components/schemas/PeersTable"
         codex:
@@ -153,12 +157,12 @@ components:
           description: Total size of availability's storage in bytes as decimal string
         duration:
           $ref: "#/components/schemas/Duration"
-        minPrice:
+        minPricePerBytePerSecond:
           type: string
-          description: Minimal price paid (in amount of tokens) for the whole hosted request's slot for the request's duration as decimal string
-        maxCollateral:
+          description: Minimal price per byte per second paid (in amount of tokens) for the hosted request's slot for the request's duration as decimal string
+        totalCollateral:
           type: string
-          description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
+          description: Total collateral (in amount of tokens) that can be used for matching requests
 
     SalesAvailabilityREAD:
       allOf:
@@ -174,8 +178,8 @@ components:
         - $ref: "#/components/schemas/SalesAvailability"
         - required:
             - totalSize
-            - minPrice
-            - maxCollateral
+            - minPricePerBytePerSecond
+            - totalCollateral
             - duration
 
     Slot:
@@ -239,16 +243,16 @@ components:
     StorageRequestCreation:
       type: object
       required:
-        - reward
+        - pricePerBytePerSecond
         - duration
         - proofProbability
-        - collateral
+        - collateralPerByte
         - expiry
       properties:
         duration:
           $ref: "#/components/schemas/Duration"
-        reward:
-          $ref: "#/components/schemas/Reward"
+        pricePerBytePerSecond:
+          $ref: "#/components/schemas/PricePerBytePerSecond"
         proofProbability:
           $ref: "#/components/schemas/ProofProbability"
         nodes:
@@ -259,16 +263,16 @@ components:
           description: Additional number of nodes on top of the `nodes` property that can be lost before pronouncing the content lost
           type: integer
           default: 0
-        collateral:
+        collateralPerByte:
           type: string
-          description: Number as decimal string that represents how much collateral is asked from hosts that wants to fill a slots
+          description: Number as decimal string that represents how much collateral per byte is asked from hosts that wants to fill a slots
         expiry:
           type: string
           description: Number as decimal string that represents expiry threshold in seconds from when the Request is submitted. When the threshold is reached and the Request does not find requested amount of nodes to host the data, the Request is voided. The number of seconds can not be higher then the Request's duration itself.
     StorageAsk:
       type: object
       required:
-        - reward
+        - pricePerBytePerSecond
       properties:
         slots:
           description: Number of slots (eq. hosts) that the Request want to have the content spread over
@@ -280,8 +284,8 @@ components:
           $ref: "#/components/schemas/Duration"
         proofProbability:
           $ref: "#/components/schemas/ProofProbability"
-        reward:
-          $ref: "#/components/schemas/Reward"
+        pricePerBytePerSecond:
+          $ref: "#/components/schemas/PricePerBytePerSecond"
         maxSlotLoss:
           type: integer
           description: Max slots that can be lost without data considered to be lost
@@ -321,6 +325,7 @@ components:
             - unknown
         error:
           type: string
+          nullable: true
           description: If Request failed, then here is presented the error message
         request:
           $ref: "#/components/schemas/StorageRequest"
@@ -367,12 +372,6 @@ components:
           nullable: true
           description: "The original mimetype of the uploaded content (optional)"
           example: image/png
-        uploadedAt:
-          type: integer
-          format: int64
-          nullable: true
-          description: "The UTC upload timestamp in seconds"
-          example: 1729244192
 
     Space:
       type: object
@@ -414,14 +413,14 @@ paths:
       description: |
         If `addrs` param is supplied, it will be used to dial the peer, otherwise the `peerId` is used
         to invoke peer discovery, if it succeeds the returned addresses will be used to dial.
-      tags: [ Node ]
+      tags: [Node]
       operationId: connectPeer
       parameters:
         - in: path
           name: peerId
           required: true
           schema:
-              $ref: "#/components/schemas/PeerId"
+            $ref: "#/components/schemas/PeerId"
           description: Peer that should be dialed.
         - in: query
           name: addrs
@@ -444,7 +443,7 @@ paths:
   "/data":
     get:
       summary: "Lists manifest CIDs stored locally in node."
-      tags: [ Data ]
+      tags: [Data]
       operationId: listData
       responses:
         "200":
@@ -464,7 +463,7 @@ paths:
           description: Well it was bad-bad
     post:
       summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
-      tags: [ Data ]
+      tags: [Data]
       operationId: upload
       parameters:
         - name: content-type
@@ -480,7 +479,7 @@ paths:
           description: The content disposition used to send the filename.
           schema:
             type: string
-            example: "attachment; filename=\"codex.png\""
+            example: 'attachment; filename="codex.png"'
       requestBody:
         content:
           application/octet-stream:
@@ -500,14 +499,14 @@ paths:
   "/data/{cid}":
     get:
       summary: "Download a file from the local node in a streaming manner. If the file is not available locally, a 404 is returned."
-      tags: [ Data ]
+      tags: [Data]
       operationId: downloadLocal
       parameters:
         - in: path
           name: cid
           required: true
           schema:
-              $ref: "#/components/schemas/Cid"
+            $ref: "#/components/schemas/Cid"
           description: File to be downloaded.
 
       responses:
@@ -528,14 +527,14 @@ paths:
   "/data/{cid}/network":
     post:
       summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
-      tags: [ Data ]
+      tags: [Data]
       operationId: downloadNetwork
       parameters:
         - in: path
           name: cid
           required: true
           schema:
-              $ref: "#/components/schemas/Cid"
+            $ref: "#/components/schemas/Cid"
           description: "File to be downloaded."
       responses:
         "200":
@@ -554,14 +553,14 @@ paths:
   "/data/{cid}/network/stream":
     get:
       summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
-      tags: [ Data ]
+      tags: [Data]
       operationId: downloadNetworkStream
       parameters:
         - in: path
           name: cid
           required: true
           schema:
-              $ref: "#/components/schemas/Cid"
+            $ref: "#/components/schemas/Cid"
           description: "File to be downloaded."
       responses:
         "200":
@@ -581,14 +580,14 @@ paths:
   "/data/{cid}/network/manifest":
     get:
       summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
-      tags: [ Data ]
+      tags: [Data]
       operationId: downloadNetworkManifest
       parameters:
         - in: path
           name: cid
           required: true
           schema:
-              $ref: "#/components/schemas/Cid"
+            $ref: "#/components/schemas/Cid"
           description: "File for which the manifest is to be downloaded."
       responses:
         "200":
@@ -607,7 +606,7 @@ paths:
   "/space":
     get:
       summary: "Gets a summary of the storage space allocation of the node."
-      tags: [ Data ]
+      tags: [Data]
       operationId: space
       responses:
         "200":
@@ -623,7 +622,7 @@ paths:
   "/sales/slots":
     get:
       summary: "Returns active slots"
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       operationId: getActiveSlots
       responses:
         "200":
@@ -641,7 +640,7 @@ paths:
   "/sales/slots/{slotId}":
     get:
       summary: "Returns active slot with id {slotId} for the host"
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       operationId: getActiveSlotById
       parameters:
         - in: path
@@ -670,7 +669,7 @@ paths:
   "/sales/availability":
     get:
       summary: "Returns storage that is for sale"
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       operationId: getAvailabilities
       responses:
         "200":
@@ -689,7 +688,7 @@ paths:
     post:
       summary: "Offers storage for sale"
       operationId: offerStorage
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       requestBody:
         content:
           application/json:
@@ -717,7 +716,7 @@ paths:
         The new parameters will be only considered for new requests.
         Existing Requests linked to this Availability will continue as is.
       operationId: updateOfferedStorage
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       parameters:
         - in: path
           name: id
@@ -749,7 +748,7 @@ paths:
       summary: "Get availability's reservations"
       description: Return's list of Reservations for ongoing Storage Requests that the node hosts.
       operationId: getReservations
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       parameters:
         - in: path
           name: id
@@ -778,7 +777,7 @@ paths:
   "/storage/request/{cid}":
     post:
       summary: "Creates a new Request for storage"
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       operationId: createStorageRequest
       parameters:
         - in: path
@@ -809,7 +808,7 @@ paths:
   "/storage/purchases":
     get:
       summary: "Returns list of purchase IDs"
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       operationId: getPurchases
       responses:
         "200":
@@ -826,7 +825,7 @@ paths:
   "/storage/purchases/{id}":
     get:
       summary: "Returns purchase details"
-      tags: [ Marketplace ]
+      tags: [Marketplace]
       operationId: getPurchase
       parameters:
         - in: path
@@ -853,7 +852,7 @@ paths:
     get:
       summary: "Get Node's SPR"
       operationId: getSPR
-      tags: [ Node ]
+      tags: [Node]
       responses:
         "200":
           description: Node's SPR
@@ -871,7 +870,7 @@ paths:
     get:
       summary: "Get Node's PeerID"
       operationId: getPeerId
-      tags: [ Node ]
+      tags: [Node]
       responses:
         "200":
           description: Node's Peer ID
@@ -886,7 +885,7 @@ paths:
   "/debug/chronicles/loglevel":
     post:
       summary: "Set log level at run time"
-      tags: [ Debug ]
+      tags: [Debug]
       operationId: setDebugLogLevel
 
       parameters:
@@ -908,7 +907,7 @@ paths:
     get:
       summary: "Gets node information"
       operationId: getDebugInfo
-      tags: [ Debug ]
+      tags: [Debug]
       responses:
         "200":
           description: Node's information

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs

@@ -14,7 +14,8 @@ namespace CodexContractsPlugin.ChainMonitor
             RequestFailedEventDTO[] failed,
             SlotFilledEventDTO[] slotFilled,
             SlotFreedEventDTO[] slotFreed,
-            SlotReservationsFullEventDTO[] slotReservationsFull
+            SlotReservationsFullEventDTO[] slotReservationsFull,
+            ProofSubmittedEventDTO[] proofSubmitted
             )
         {
             BlockInterval = blockInterval;
@@ -25,8 +26,8 @@ namespace CodexContractsPlugin.ChainMonitor
             SlotFilled = slotFilled;
             SlotFreed = slotFreed;
             SlotReservationsFull = slotReservationsFull;
-
-            All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull);
+            ProofSubmitted = proofSubmitted;
+            All = ConcatAll<IHasBlock>(requests, fulfilled, cancelled, failed, slotFilled, SlotFreed, SlotReservationsFull, ProofSubmitted);
         }
 
         public BlockInterval BlockInterval { get; }
@@ -37,6 +38,7 @@ namespace CodexContractsPlugin.ChainMonitor
         public SlotFilledEventDTO[] SlotFilled { get; }
         public SlotFreedEventDTO[] SlotFreed { get; }
         public SlotReservationsFullEventDTO[] SlotReservationsFull { get; }
+        public ProofSubmittedEventDTO[] ProofSubmitted { get; }
         public IHasBlock[] All { get; }
 
         public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval)
@@ -59,7 +61,8 @@ namespace CodexContractsPlugin.ChainMonitor
                 events.GetRequestFailedEvents(),
                 events.GetSlotFilledEvents(),
                 events.GetSlotFreedEvents(),
-                events.GetSlotReservationsFull()
+                events.GetSlotReservationsFullEvents(),
+                events.GetProofSubmittedEvents()
             );
         }
 

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs

@@ -1,7 +1,7 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
-using NethereumWorkflow.BlockUtils;
 using System.Numerics;
 using Utils;
 
@@ -17,7 +17,7 @@ namespace CodexContractsPlugin.ChainMonitor
         void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
         void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
         void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
-
+        void OnProofSubmitted(BlockTimeEntry block, string id);
         void OnError(string msg);
     }
 
@@ -39,17 +39,21 @@ namespace CodexContractsPlugin.ChainMonitor
         private readonly ILog log;
         private readonly ICodexContracts contracts;
         private readonly IChainStateChangeHandler handler;
+        private readonly bool doProofPeriodMonitoring;
 
-        public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc)
+        public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc, bool doProofPeriodMonitoring)
         {
             this.log = new LogPrefixer(log, "(ChainState) ");
             this.contracts = contracts;
             handler = changeHandler;
+            this.doProofPeriodMonitoring = doProofPeriodMonitoring;
             TotalSpan = new TimeRange(startUtc, startUtc);
+            PeriodMonitor = new PeriodMonitor(this.log, contracts);
         }
 
         public TimeRange TotalSpan { get; private set; }
         public IChainStateRequest[] Requests => requests.ToArray();
+        public PeriodMonitor PeriodMonitor { get; }
 
         public int Update()
         {
@@ -88,11 +92,18 @@ namespace CodexContractsPlugin.ChainMonitor
             {
                 var blockEvents = events.All.Where(e => e.Block.BlockNumber == b).ToArray();
                 ApplyEvents(b, blockEvents, eventUtc);
+                UpdatePeriodMonitor(b, eventUtc);
 
                 eventUtc += spanPerBlock;
             }
         }
 
+        private void UpdatePeriodMonitor(ulong blockNumber, DateTime eventUtc)
+        {
+            if (!doProofPeriodMonitoring) return;
+            PeriodMonitor.Update(blockNumber, eventUtc, Requests);
+        }
+
         private void ApplyEvents(ulong blockNumber, IHasBlock[] blockEvents, DateTime eventsUtc)
         {
             foreach (var e in blockEvents)
@@ -165,6 +176,13 @@ namespace CodexContractsPlugin.ChainMonitor
             handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
         }
 
+        private void ApplyEvent(ProofSubmittedEventDTO @event)
+        {
+            var id = Base58.Encode(@event.Id);
+            log.Log($"[{@event.Block.BlockNumber}] Proof submitted (id:{id})");
+            handler.OnProofSubmitted(@event.Block, id);
+        }
+
         private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc)
         {
             foreach (var r in requests)

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs

@@ -1,5 +1,6 @@
-using GethPlugin;
+using BlockchainUtils;
 using System.Numerics;
+using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
@@ -56,5 +57,10 @@ namespace CodexContractsPlugin.ChainMonitor
         {
             foreach (var handler in Handlers) handler.OnError(msg);
         }
+
+        public void OnProofSubmitted(BlockTimeEntry block, string id)
+        {
+            foreach (var handler in Handlers) handler.OnProofSubmitted(block, id);
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs

@@ -1,6 +1,6 @@
 using CodexContractsPlugin.Marketplace;
-using GethPlugin;
 using Logging;
+using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
 {

ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs

@@ -1,5 +1,6 @@
-using GethPlugin;
+using BlockchainUtils;
 using System.Numerics;
+using Utils;
 
 namespace CodexContractsPlugin.ChainMonitor
 {
@@ -40,5 +41,9 @@ namespace CodexContractsPlugin.ChainMonitor
         public void OnError(string msg)
         {
         }
+
+        public void OnProofSubmitted(BlockTimeEntry block, string id)
+        {
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/ChainMonitor/PeriodMonitor.cs

@@ -0,0 +1,99 @@
+using Logging;
+using Utils;
+
+namespace CodexContractsPlugin.ChainMonitor
+{
+    public class PeriodMonitor
+    {
+        private readonly ILog log;
+        private readonly ICodexContracts contracts;
+        private readonly List<PeriodReport> reports = new List<PeriodReport>();
+        private ulong? currentPeriod = null;
+
+        public PeriodMonitor(ILog log, ICodexContracts contracts)
+        {
+            this.log = log;
+            this.contracts = contracts;
+        }
+
+        public void Update(ulong blockNumber, DateTime eventUtc, IChainStateRequest[] requests)
+        {
+            var period = contracts.GetPeriodNumber(eventUtc);
+            if (!currentPeriod.HasValue)
+            {
+                currentPeriod = period;
+                return;
+            }
+            if (period == currentPeriod.Value) return;
+
+            CreateReportForPeriod(blockNumber - 1, currentPeriod.Value, requests);
+            currentPeriod = period;
+        }
+
+        public PeriodReport[] GetAndClearReports()
+        {
+            var result = reports.ToArray();
+            reports.Clear();
+            return result;
+        }
+
+        private void CreateReportForPeriod(ulong lastBlockInPeriod, ulong periodNumber, IChainStateRequest[] requests)
+        {
+            log.Log("Creating report for period " + periodNumber);
+
+            ulong total = 0;
+            ulong required = 0;
+            var missed = new List<PeriodProofMissed>();
+            foreach (var request in requests)
+            {
+                for (ulong slotIndex = 0; slotIndex < request.Request.Ask.Slots; slotIndex++)
+                {
+                    var state = contracts.GetProofState(request.Request, slotIndex, lastBlockInPeriod, periodNumber);
+
+                    total++;
+                    if (state.Required)
+                    {
+                        required++;
+                        if (state.Missing)
+                        {
+                            var idx = Convert.ToInt32(slotIndex);
+                            var host = request.Hosts.GetHost(idx);
+                            missed.Add(new PeriodProofMissed(host, request, idx));
+                        }
+                    }
+                }
+            }
+            reports.Add(new PeriodReport(periodNumber, total, required, missed.ToArray()));
+        }
+    }
+
+    public class PeriodReport
+    {
+        public PeriodReport(ulong periodNumber, ulong totalNumSlots, ulong totalProofsRequired, PeriodProofMissed[] missedProofs)
+        {
+            PeriodNumber = periodNumber;
+            TotalNumSlots = totalNumSlots;
+            TotalProofsRequired = totalProofsRequired;
+            MissedProofs = missedProofs;
+        }
+
+        public ulong PeriodNumber { get; }
+        public ulong TotalNumSlots { get; }
+        public ulong TotalProofsRequired { get; }
+        public PeriodProofMissed[] MissedProofs { get; }
+    }
+
+    public class PeriodProofMissed
+    {
+        public PeriodProofMissed(EthAddress? host, IChainStateRequest request, int slotIndex)
+        {
+            Host = host;
+            Request = request;
+            SlotIndex = slotIndex;
+        }
+
+        public EthAddress? Host { get; }
+        public IChainStateRequest Request { get; }
+        public int SlotIndex { get; }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -1,7 +1,11 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.ABI;
+using Nethereum.ABI.FunctionEncoding.Attributes;
+using Nethereum.Contracts;
+using Nethereum.Hex.HexConvertors.Extensions;
 using Nethereum.Util;
 using NethereumWorkflow;
 using Newtonsoft.Json;
@@ -24,6 +28,21 @@ namespace CodexContractsPlugin
         ICodexContractsEvents GetEvents(BlockInterval blockInterval);
         EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
         RequestState GetRequestState(Request request);
+        ulong GetPeriodNumber(DateTime utc);
+        void WaitUntilNextPeriod();
+        ProofState GetProofState(Request storageRequest, decimal slotIndex, ulong blockNumber, ulong period);
+    }
+
+    public class ProofState
+    {
+        public ProofState(bool required, bool missing)
+        {
+            Required = required;
+            Missing = missing;
+        }
+
+        public bool Required { get; }
+        public bool Missing { get; }
     }
 
     [JsonConverter(typeof(StringEnumConverter))]
@@ -86,19 +105,23 @@ namespace CodexContractsPlugin
             return new CodexContractsEvents(log, gethNode, Deployment, blockInterval);
         }
 
-        public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
+        public byte[] GetSlotId(Request request, decimal slotIndex)
         {
             var encoder = new ABIEncode();
             var encoded = encoder.GetABIEncoded(
-                new ABIValue("bytes32", storageRequest.RequestId),
+                new ABIValue("bytes32", request.RequestId),
                 new ABIValue("uint256", slotIndex.ToBig())
             );
 
-            var hashed = Sha3Keccack.Current.CalculateHash(encoded);
+            return Sha3Keccack.Current.CalculateHash(encoded);
+        }
 
+        public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
+        {
+            var slotId = GetSlotId(storageRequest, slotIndex);
             var func = new GetHostFunction
             {
-                SlotId = hashed
+                SlotId = slotId
             };
             var address = gethNode.Call<GetHostFunction, string>(Deployment.MarketplaceAddress, func);
             if (string.IsNullOrEmpty(address)) return null;
@@ -114,6 +137,70 @@ namespace CodexContractsPlugin
             return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
         }
 
+        public ulong GetPeriodNumber(DateTime utc)
+        {
+            DateTimeOffset utco = DateTime.SpecifyKind(utc, DateTimeKind.Utc);
+            var now = utco.ToUnixTimeSeconds();
+            var periodSeconds = (int)Deployment.Config.Proofs.Period;
+            var result = now / periodSeconds;
+            return Convert.ToUInt64(result);
+        }
+
+        public void WaitUntilNextPeriod()
+        {
+            log.Log("Waiting until next proof period...");
+            var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
+            var periodSeconds = (int)Deployment.Config.Proofs.Period;
+            var secondsLeft = now % periodSeconds;
+            Thread.Sleep(TimeSpan.FromSeconds(secondsLeft + 1));
+        }
+
+        public ProofState GetProofState(Request storageRequest, decimal slotIndex, ulong blockNumber, ulong period)
+        {
+            var slotId = GetSlotId(storageRequest, slotIndex);
+
+            var required = IsProofRequired(slotId, blockNumber);
+            if (!required) return new ProofState(false, false);
+
+            var missing = IsProofMissing(slotId, blockNumber, period);
+            return new ProofState(required, missing);
+        }
+
+        private bool IsProofRequired(byte[] slotId, ulong blockNumber)
+        {
+            var func = new IsProofRequiredFunction
+            {
+                Id = slotId
+            };
+            var result = gethNode.Call<IsProofRequiredFunction, IsProofRequiredOutputDTO>(Deployment.MarketplaceAddress, func, blockNumber);
+            return result.ReturnValue1;
+        }
+
+        private bool IsProofMissing(byte[] slotId, ulong blockNumber, ulong period)
+        {
+            try
+            {
+                var funcB = new MarkProofAsMissingFunction
+                {
+                    SlotId = slotId,
+                    Period = period
+                };
+                gethNode.Call<MarkProofAsMissingFunction>(Deployment.MarketplaceAddress, funcB, blockNumber);
+            }
+            catch (AggregateException exc)
+            {
+                if (exc.InnerExceptions.Count == 1)
+                {
+                    if (exc.InnerExceptions[0].GetType() == typeof(SmartContractCustomErrorRevertException))
+                    {
+                        return false;
+                    }
+                }
+                throw;
+            }
+            return true;
+        }
+
         private ContractInteractions StartInteraction()
         {
             return new ContractInteractions(log, gethNode);

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -1,7 +1,6 @@
 using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Recipe;
-using Logging;
 
 namespace CodexContractsPlugin
 {

ProjectPlugins/CodexContractsPlugin/CodexContractsDeployment.cs

@@ -1,14 +1,18 @@
-namespace CodexContractsPlugin
+using CodexContractsPlugin.Marketplace;
+
+namespace CodexContractsPlugin
 {
     public class CodexContractsDeployment
     {
-        public CodexContractsDeployment(string marketplaceAddress, string abi, string tokenAddress)
+        public CodexContractsDeployment(MarketplaceConfig config, string marketplaceAddress, string abi, string tokenAddress)
         {
+            Config = config;
             MarketplaceAddress = marketplaceAddress;
             Abi = abi;
             TokenAddress = tokenAddress;
         }
 
+        public MarketplaceConfig Config { get; }
         public string MarketplaceAddress { get; }
         public string Abi { get; }
         public string TokenAddress { get; }

ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs

@@ -1,9 +1,9 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.Contracts;
 using Nethereum.Hex.HexTypes;
-using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -17,7 +17,8 @@ namespace CodexContractsPlugin
         RequestFailedEventDTO[] GetRequestFailedEvents();
         SlotFilledEventDTO[] GetSlotFilledEvents();
         SlotFreedEventDTO[] GetSlotFreedEvents();
-        SlotReservationsFullEventDTO[] GetSlotReservationsFull();
+        SlotReservationsFullEventDTO[] GetSlotReservationsFullEvents();
+        ProofSubmittedEventDTO[] GetProofSubmittedEvents();
     }
 
     public class CodexContractsEvents : ICodexContractsEvents
@@ -86,12 +87,18 @@ namespace CodexContractsPlugin
             return events.Select(SetBlockOnEvent).ToArray();
         }
 
-        public SlotReservationsFullEventDTO[] GetSlotReservationsFull()
+        public SlotReservationsFullEventDTO[] GetSlotReservationsFullEvents()
         {
             var events = gethNode.GetEvents<SlotReservationsFullEventDTO>(deployment.MarketplaceAddress, BlockInterval);
             return events.Select(SetBlockOnEvent).ToArray();
         }
 
+        public ProofSubmittedEventDTO[] GetProofSubmittedEvents()
+        {
+            var events = gethNode.GetEvents<ProofSubmittedEventDTO>(deployment.MarketplaceAddress, BlockInterval);
+            return events.Select(SetBlockOnEvent).ToArray();
+        }
+
         private T SetBlockOnEvent<T>(EventLog<T> e) where T : IHasBlock
         {
             var result = e.Event;

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -4,6 +4,7 @@ using GethPlugin;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Logging;
+using Newtonsoft.Json;
 using Utils;
 
 namespace CodexContractsPlugin
@@ -34,6 +35,7 @@ namespace CodexContractsPlugin
             try
             {
                 var result = DeployContract(container, workflow, gethNode);
+
                 workflow.Stop(containers, waitTillStopped: false);
                 Log("Container stopped.");
                 return result;
@@ -75,9 +77,20 @@ namespace CodexContractsPlugin
 
             Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract));
 
-            Log("Synced. Codex SmartContracts deployed.");
+            Log("Synced. Codex SmartContracts deployed. Getting configuration...");
 
-            return new CodexContractsDeployment(marketplaceAddress, abi, tokenAddress);
+            var config = GetMarketplaceConfiguration(marketplaceAddress, gethNode);
+
+            Log("Got config: " + JsonConvert.SerializeObject(config));
+
+            return new CodexContractsDeployment(config, marketplaceAddress, abi, tokenAddress);
+        }
+
+        private MarketplaceConfig GetMarketplaceConfiguration(string marketplaceAddress, IGethNode gethNode)
+        {
+            var func = new ConfigurationFunctionBase();
+            var response = gethNode.Call<ConfigurationFunctionBase, ConfigurationOutputDTO>(marketplaceAddress, func);
+            return response.ReturnValue1;
         }
 
         private void EnsureCompatbility(string abi, string bytecode)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -1,11 +1,12 @@
-using CodexContractsPlugin.Marketplace;
+using BlockchainUtils;
+using CodexContractsPlugin.Marketplace;
 using GethPlugin;
 using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.Hex.HexConvertors.Extensions;
-using NethereumWorkflow;
 using System.Numerics;
+using Utils;
 
 namespace CodexContractsPlugin
 {
@@ -23,9 +24,8 @@ namespace CodexContractsPlugin
         public string GetTokenAddress(string marketplaceAddress)
         {
             log.Debug(marketplaceAddress);
-            var function = new GetTokenFunction();
-
-            return gethNode.Call<GetTokenFunction, string>(marketplaceAddress, function);
+            var function = new TokenFunctionBase();
+            return gethNode.Call<TokenFunctionBase, string>(marketplaceAddress, function);
         }
 
         public string GetTokenName(string tokenAddress)
@@ -111,11 +111,6 @@ namespace CodexContractsPlugin
         }
     }
 
-    [Function("token", "address")]
-    public class GetTokenFunction : FunctionMessage
-    {
-    }
-
     [Function("name", "string")]
     public class GetTokenNameFunction : FunctionMessage
     {

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -1,7 +1,7 @@
 #pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
-using GethPlugin;
-using NethereumWorkflow.BlockUtils;
+using BlockchainUtils;
 using Newtonsoft.Json;
+using Utils;
 
 namespace CodexContractsPlugin.Marketplace
 {
@@ -69,5 +69,11 @@ namespace CodexContractsPlugin.Marketplace
         [JsonIgnore]
         public BlockTimeEntry Block { get; set; }
     }
+
+    public partial class ProofSubmittedEventDTO : IHasBlock
+    {
+        [JsonIgnore]
+        public BlockTimeEntry Block { get; set; }
+    }
 }
 #pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -10,7 +10,7 @@ namespace CodexPlugin
     public class ApiChecker
     {
         // <INSERT-OPENAPI-YAML-HASH>
-        private const string OpenApiYamlHash = "34-B5-DA-26-40-76-B8-D8-8E-7D-9C-17-85-C6-B0-63-55-8D-C6-01-0B-96-BB-7C-BD-53-E5-32-07-ED-29-92";
+        private const string OpenApiYamlHash = "00-7D-C3-0B-D2-23-D6-6C-CA-C2-43-D0-9B-B4-63-FC-4F-FE-23-9F-B8-82-5F-3B-3F-6B-4F-1F-11-E9-48-16";
         private const string OpenApiFilePath = "/codex/openapi.yaml";
         private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
 
@@ -82,12 +82,12 @@ namespace CodexPlugin
         private void OverwriteOpenApiYaml(string containerApi)
         {
             Log("API compatibility check failed. Updating CodexPlugin...");
-            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin", "openapi.yaml");
-            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexPlugin/openapi.yaml. Expected: " + openApiFilePath);
+            var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexClient", "openapi.yaml");
+            if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexClient/openapi.yaml. Expected: " + openApiFilePath);
 
             File.Delete(openApiFilePath);
             File.WriteAllText(openApiFilePath, containerApi);
-            Log("CodexPlugin/openapi.yaml has been updated.");
+            Log("CodexClient/openapi.yaml has been updated.");
         }
 
         private string Hash(string file)

ProjectPlugins/CodexPlugin/BinaryCodexStarter.cs

@@ -0,0 +1,156 @@
+using CodexClient;
+using Core;
+using Utils;
+using System.Diagnostics;
+
+namespace CodexPlugin
+{
+    public class BinaryCodexStarter : ICodexStarter
+    {
+        private readonly IPluginTools pluginTools;
+        private readonly ProcessControlMap processControlMap;
+        private readonly static NumberSource numberSource = new NumberSource(1);
+        private readonly static FreePortFinder freePortFinder = new FreePortFinder();
+        private readonly static object _lock = new object();
+        private readonly static string dataParentDir = "codex_disttest_datadirs";
+        private readonly static CodexExePath codexExePath = new CodexExePath();
+
+        static BinaryCodexStarter()
+        {
+            StopAllCodexProcesses();
+            DeleteParentDataDir();
+        }
+
+        public BinaryCodexStarter(IPluginTools pluginTools, ProcessControlMap processControlMap)
+        {
+            this.pluginTools = pluginTools;
+            this.processControlMap = processControlMap;
+        }
+
+        public ICodexInstance[] BringOnline(CodexSetup codexSetup)
+        {
+            lock (_lock)
+            {
+                LogSeparator();
+                Log($"Starting {codexSetup.Describe()}...");
+
+                return StartCodexBinaries(codexSetup, codexSetup.NumberOfNodes);
+            }
+        }
+
+        public void Decommission()
+        {
+            lock (_lock)
+            {
+                processControlMap.StopAll();
+            }
+        }
+
+        private ICodexInstance[] StartCodexBinaries(CodexStartupConfig startupConfig, int numberOfNodes)
+        {
+            var result = new List<ICodexInstance>();
+            for (var i = 0; i < numberOfNodes; i++)
+            {
+                result.Add(StartBinary(startupConfig));
+            }
+
+            return result.ToArray();
+        }
+
+        private ICodexInstance StartBinary(CodexStartupConfig config)
+        {
+            var name = GetName(config);
+            var dataDir = Path.Combine(dataParentDir, $"datadir_{numberSource.GetNextNumber()}");
+            var pconfig = new CodexProcessConfig(name, freePortFinder, dataDir);
+            Log(pconfig);
+
+            var factory = new CodexProcessRecipe(pconfig, codexExePath);
+            var recipe = factory.Initialize(config);
+
+            var startInfo = new ProcessStartInfo(
+                fileName: recipe.Cmd,
+                arguments: recipe.Args
+            );
+            //startInfo.UseShellExecute = true;
+            startInfo.RedirectStandardOutput = true;
+            startInfo.RedirectStandardError = true;
+
+            var process = Process.Start(startInfo);
+            if (process == null || process.HasExited)
+            {
+                throw new Exception("Failed to start");
+            }
+
+            var local = "localhost";
+            var instance = new CodexInstance(
+                name: name,
+                imageName: "binary",
+                startUtc: DateTime.UtcNow,
+                discoveryEndpoint: new Address("Disc", pconfig.LocalIpAddrs.ToString(), pconfig.DiscPort),
+                apiEndpoint: new Address("Api", "http://" + local, pconfig.ApiPort),
+                listenEndpoint: new Address("Listen", local, pconfig.ListenPort),
+                ethAccount: null,
+                metricsEndpoint: null
+            );
+
+            var pc = new BinaryProcessControl(pluginTools.GetLog(), process, pconfig);
+            processControlMap.Add(instance, pc);
+
+            return instance;
+        }
+
+        private string GetName(CodexStartupConfig config)
+        {
+            if (!string.IsNullOrEmpty(config.NameOverride))
+            {
+                return config.NameOverride + "_" + numberSource.GetNextNumber();
+            }
+            return "codex_" + numberSource.GetNextNumber();
+        }
+
+        private void LogSeparator()
+        {
+            Log("----------------------------------------------------------------------------");
+        }
+
+        private void Log(CodexProcessConfig pconfig)
+        {
+            Log(
+                "NodeConfig:Name=" + pconfig.Name +
+                "ApiPort=" + pconfig.ApiPort +
+                "DiscPort=" + pconfig.DiscPort +
+                "ListenPort=" + pconfig.ListenPort +
+                "DataDir=" + pconfig.DataDir
+            );
+        }
+
+        private void Log(string message)
+        {
+            pluginTools.GetLog().Log(message);
+        }
+
+        private static void DeleteParentDataDir()
+        {
+            if (Directory.Exists(dataParentDir))
+            {
+                Directory.Delete(dataParentDir, true);
+            }
+        }
+
+        private static void StopAllCodexProcesses()
+        {
+            var processes = Process.GetProcesses();
+            var codexes = processes.Where(p =>
+                p.ProcessName.ToLowerInvariant() == "codex" &&
+                p.MainModule != null &&
+                p.MainModule.FileName == codexExePath.Get()
+            ).ToArray();
+
+            foreach (var c in codexes)
+            {
+                c.Kill();
+                c.WaitForExit();
+            }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/BinaryProcessControl.cs

@@ -0,0 +1,93 @@
+using System.Diagnostics;
+using CodexClient;
+using Logging;
+
+namespace CodexPlugin
+{
+    public class BinaryProcessControl : IProcessControl
+    {
+        private readonly LogFile logFile;
+        private readonly Process process;
+        private readonly CodexProcessConfig config;
+        private List<string> logBuffer = new List<string>();
+        private readonly object bufferLock = new object();
+        private readonly List<Task> streamTasks = new List<Task>();
+        private bool running;
+
+        public BinaryProcessControl(ILog log, Process process, CodexProcessConfig config)
+        {
+            logFile = log.CreateSubfile(config.Name);
+
+            running = true;
+            this.process = process;
+            this.config = config;
+            streamTasks.Add(Task.Run(() => ReadProcessStream(process.StandardOutput)));
+            streamTasks.Add(Task.Run(() => ReadProcessStream(process.StandardError)));
+            streamTasks.Add(Task.Run(() => WriteLog()));
+        }
+
+        private void ReadProcessStream(StreamReader reader)
+        {
+            while (running)
+            {
+                var line = reader.ReadLine();
+                if (!string.IsNullOrEmpty(line))
+                {
+                    lock (bufferLock)
+                    {
+                        logBuffer.Add(line);
+                    }
+                }
+            }
+        }
+
+        private void WriteLog()
+        {
+            while (running || logBuffer.Count > 0)
+            {
+                if (logBuffer.Count > 0)
+                {
+                    List<string> lines = null!;
+                    lock (bufferLock)
+                    {
+                        lines = logBuffer;
+                        logBuffer = new List<string>();
+                    }
+
+                    logFile.WriteRawMany(lines);
+                }
+                else Thread.Sleep(100);
+            }
+        }
+
+        public void DeleteDataDirFolder()
+        {
+            if (!Directory.Exists(config.DataDir)) throw new Exception("datadir not found");
+            Directory.Delete(config.DataDir, true);
+        }
+
+        public IDownloadedLog DownloadLog(LogFile file)
+        {
+            return new DownloadedLog(logFile, config.Name);
+        }
+
+        public bool HasCrashed()
+        {
+            return process.HasExited;
+        }
+
+        public void Stop(bool waitTillStopped)
+        {
+            running = false;
+            process.Kill();
+
+            if (waitTillStopped)
+            {
+                process.WaitForExit();
+                foreach (var t in streamTasks) t.Wait();
+            }
+
+            DeleteDataDirFolder();
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -1,291 +0,0 @@
-using CodexOpenApi;
-using Core;
-using KubernetesWorkflow;
-using KubernetesWorkflow.Types;
-using Logging;
-using Newtonsoft.Json;
-using Utils;
-
-namespace CodexPlugin
-{
-    public class CodexAccess
-    {
-        private readonly ILog log;
-        private readonly IPluginTools tools;
-        private readonly Mapper mapper = new Mapper();
-
-        public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher)
-        {
-            this.tools = tools;
-            log = tools.GetLog();
-            Container = container;
-            CrashWatcher = crashWatcher;
-
-            CrashWatcher.Start();
-        }
-
-        public RunningPod Container { get; }
-        public CrashWatcher CrashWatcher { get; }
-
-        public DebugInfo GetDebugInfo()
-        {
-            return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
-        }
-
-        public DebugPeer GetDebugPeer(string peerId)
-        {
-            // Cannot use openAPI: debug/peer endpoint is not specified there.
-            return CrashCheck(() =>
-            {
-                var endpoint = GetEndpoint();
-                var str = endpoint.HttpGetString($"debug/peer/{peerId}");
-
-                if (str.ToLowerInvariant() == "unable to find peer!")
-                {
-                    return new DebugPeer
-                    {
-                        IsPeerFound = false
-                    };
-                }
-
-                var result = endpoint.Deserialize<DebugPeer>(str);
-                result.IsPeerFound = true;
-                return result;
-            });
-        }
-
-        public void ConnectToPeer(string peerId, string[] peerMultiAddresses)
-        {
-            OnCodex(api =>
-            {
-                Time.Wait(api.ConnectPeerAsync(peerId, peerMultiAddresses));
-                return Task.FromResult(string.Empty);
-            });
-        }
-
-        public string UploadFile(UploadInput uploadInput, Action<Failure> onFailure)
-        {
-            return OnCodex(
-                api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream),
-                CreateRetryConfig(nameof(UploadFile), onFailure));
-        }
-
-        public Stream DownloadFile(string contentId, Action<Failure> onFailure)
-        {
-            var fileResponse = OnCodex(
-                api => api.DownloadNetworkStreamAsync(contentId),
-                CreateRetryConfig(nameof(DownloadFile), onFailure));
-
-            if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
-            return fileResponse.Stream;
-        }
-
-        public LocalDatasetList LocalFiles()
-        {
-            // API for listData mismatches.
-            //return mapper.Map(OnCodex(api => api.ListDataAsync()));
-
-            return mapper.Map(CrashCheck(() =>
-            {
-                var endpoint = GetEndpoint();
-                return Time.Retry(() =>
-                {
-                    var str = endpoint.HttpGetString("data");
-                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
-                    return JsonConvert.DeserializeObject<LocalDatasetListJson>(str)!;
-                }, nameof(LocalFiles));
-            }));
-
-        }
-
-        public StorageAvailability SalesAvailability(StorageAvailability request)
-        {
-            var body = mapper.Map(request);
-            var read = OnCodex(api => api.OfferStorageAsync(body));
-            return mapper.Map(read);
-        }
-
-        public StorageAvailability[] GetAvailabilities()
-        {
-            var collection = OnCodex(api => api.GetAvailabilitiesAsync());
-            return mapper.Map(collection);
-        }
-
-        public string RequestStorage(StoragePurchaseRequest request)
-        {
-            var body = mapper.Map(request);
-            return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
-        }
-
-        public CodexSpace Space()
-        {
-            var space = OnCodex(api => api.SpaceAsync());
-            return mapper.Map(space);
-        }
-
-        public StoragePurchase GetPurchaseStatus(string purchaseId)
-        {
-            return CrashCheck(() =>
-            {
-                var endpoint = GetEndpoint();
-                return Time.Retry(() =>
-                {
-                    var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
-                    if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
-                    return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
-                }, nameof(GetPurchaseStatus));
-            });
-
-            // TODO: current getpurchase api does not line up with its openapi spec.
-            // return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
-        }
-
-        public string GetName()
-        {
-            return Container.Name;
-        }
-
-        public PodInfo GetPodInfo()
-        {
-            var workflow = tools.CreateWorkflow();
-            return workflow.GetPodInfo(Container);
-        }
-
-        public void DeleteRepoFolder()
-        {
-            try
-            {
-                var containerNumber = Container.Containers.First().Recipe.Number;
-                var dataDir = $"datadir{containerNumber}";
-                var workflow = tools.CreateWorkflow();
-                workflow.ExecuteCommand(Container.Containers.First(), "rm", "-Rfv", $"/codex/{dataDir}/repo");
-                Log("Deleted repo folder.");
-            }
-            catch (Exception e)
-            {
-                Log("Unable to delete repo folder: " + e);
-            }
-        }
-
-        private T OnCodex<T>(Func<CodexApi, Task<T>> action)
-        {
-            var result = tools.CreateHttp(GetHttpId(), CheckContainerCrashed).OnClient(client => CallCodex(client, action));
-            return result;
-        }
-
-        private T OnCodex<T>(Func<CodexApi, Task<T>> action, Retry retry)
-        {
-            var result = tools.CreateHttp(GetHttpId(), CheckContainerCrashed).OnClient(client => CallCodex(client, action), retry);
-            return result;
-        }
-
-        private T CallCodex<T>(HttpClient client, Func<CodexApi, Task<T>> action)
-        {
-            var address = GetAddress();
-            var api = new CodexApi(client);
-            api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
-            return CrashCheck(() => Time.Wait(action(api)));
-        }
-
-        private T CrashCheck<T>(Func<T> action)
-        {
-            try
-            {
-                return action();
-            }
-            finally
-            {
-                CrashWatcher.HasContainerCrashed();
-            }
-        }
-
-        private IEndpoint GetEndpoint()
-        {
-            return tools
-                .CreateHttp(GetHttpId(), CheckContainerCrashed)
-                .CreateEndpoint(GetAddress(), "/api/codex/v1/", Container.Name);
-        }
-
-        private Address GetAddress()
-        {
-            return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
-        }
-
-        private string GetHttpId()
-        {
-            return GetAddress().ToString();
-        }
-
-        private void CheckContainerCrashed(HttpClient client)
-        {
-            if (CrashWatcher.HasContainerCrashed()) throw new Exception($"Container {GetName()} has crashed.");
-        }
-
-        private Retry CreateRetryConfig(string description, Action<Failure> onFailure)
-        {
-            var timeSet = tools.TimeSet;
-
-            return new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), failure =>
-            {
-                onFailure(failure);
-                Investigate(failure, timeSet);
-            });
-        }
-
-        private void Investigate(Failure failure, ITimeSet timeSet)
-        {
-            Log($"Retry {failure.TryNumber} took {Time.FormatDuration(failure.Duration)} and failed with '{failure.Exception}'. " +
-                $"(HTTP timeout = {Time.FormatDuration(timeSet.HttpCallTimeout())}) " +
-                $"Checking if node responds to debug/info...");
-
-            try
-            {
-                var debugInfo = GetDebugInfo();
-                if (string.IsNullOrEmpty(debugInfo.Spr))
-                {
-                    Log("Did not get value debug/info response.");
-                    Throw(failure);
-                }
-                else
-                {
-                    Log("Got valid response from debug/info.");
-                }
-            }
-            catch (Exception ex)
-            {
-                Log("Got exception from debug/info call: " + ex);
-                Throw(failure);
-            }
-
-            if (failure.Duration < timeSet.HttpCallTimeout())
-            {
-                Log("Retry failed within HTTP timeout duration.");
-                Throw(failure);
-            }
-        }
-
-        private void Throw(Failure failure)
-        {
-            throw failure.Exception;
-        }
-
-        private void Log(string msg)
-        {
-            log.Log($"{GetName()} {msg}");
-        }
-    }
-
-    public class UploadInput
-    {
-        public UploadInput(string contentType, string contentDisposition, FileStream fileStream)
-        {
-            ContentType = contentType;
-            ContentDisposition = contentDisposition;
-            FileStream = fileStream;
-        }
-
-        public string ContentType { get; }
-        public string ContentDisposition { get; }
-        public FileStream FileStream { get; }
-    }
-}

ProjectPlugins/CodexPlugin/CodexContainerProcessControl.cs

@@ -0,0 +1,70 @@
+using CodexClient;
+using Core;
+using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
+using Logging;
+
+namespace CodexPlugin
+{
+    public class CodexContainerProcessControl : IProcessControl
+    {
+        private readonly IPluginTools tools;
+        private readonly RunningPod pod;
+        private readonly Action onStop;
+        private readonly ContainerCrashWatcher crashWatcher;
+
+        public CodexContainerProcessControl(IPluginTools tools, RunningPod pod, Action onStop)
+        {
+            this.tools = tools;
+            this.pod = pod;
+            this.onStop = onStop;
+
+            crashWatcher = tools.CreateWorkflow().CreateCrashWatcher(pod.Containers.Single());
+            crashWatcher.Start();
+        }
+
+        public void Stop(bool waitTillStopped)
+        {
+            Log($"Stopping node...");
+            crashWatcher.Stop();
+            var workflow = tools.CreateWorkflow();
+            workflow.Stop(pod, waitTillStopped);
+            onStop();
+            Log("Stopped.");
+        }
+
+        public IDownloadedLog DownloadLog(LogFile file)
+        {
+            var workflow = tools.CreateWorkflow();
+            return workflow.DownloadContainerLog(pod.Containers.Single());
+        }
+
+        public void DeleteDataDirFolder()
+        {
+            var container = pod.Containers.Single();
+
+            try
+            {
+                var dataDirVar = container.Recipe.EnvVars.Single(e => e.Name == "CODEX_DATA_DIR");
+                var dataDir = dataDirVar.Value;
+                var workflow = tools.CreateWorkflow();
+                workflow.ExecuteCommand(container, "rm", "-Rfv", $"/codex/{dataDir}/repo");
+                Log("Deleted repo folder.");
+            }
+            catch (Exception e)
+            {
+                Log("Unable to delete repo folder: " + e);
+            }
+        }
+
+        public bool HasCrashed()
+        {
+            return crashWatcher.HasCrashed();
+        }
+
+        private void Log(string message)
+        {
+            tools.GetLog().Log(message);
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -1,4 +1,5 @@
-using CodexContractsPlugin;
+using CodexClient;
+using CodexContractsPlugin;
 using GethPlugin;
 using KubernetesWorkflow.Types;
 
@@ -9,7 +10,7 @@ namespace CodexPlugin
         public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
             CodexContractsDeployment codexContractsDeployment, RunningPod? prometheusContainer,
             RunningPod? discordBotContainer, DeploymentMetadata metadata,
-            String id)
+            string id)
         {
             Id = id;
             CodexInstances = codexInstances;
@@ -20,7 +21,7 @@ namespace CodexPlugin
             Metadata = metadata;
         }
 
-        public String Id { get; }
+        public string Id { get; }
         public CodexInstance[] CodexInstances { get; }
         public GethDeployment GethDeployment { get; }
         public CodexContractsDeployment CodexContractsDeployment { get; }
@@ -29,18 +30,6 @@ namespace CodexPlugin
         public DeploymentMetadata Metadata { get; }
     }
 
-    public class CodexInstance
-    {
-        public CodexInstance(RunningPod pod, DebugInfo info)
-        {
-            Pod = pod;
-            Info = info;
-        }
-
-        public RunningPod Pod { get; }
-        public DebugInfo Info { get; }
-    }
-
     public class DeploymentMetadata
     {
         public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace,

ProjectPlugins/CodexPlugin/CodexExePath.cs

@@ -0,0 +1,29 @@
+namespace CodexPlugin
+{
+    public class CodexExePath
+    {
+        private readonly string[] paths = [
+            Path.Combine("d:", "Dev", "nim-codex", "build", "codex.exe"),
+            Path.Combine("c:", "Projects", "nim-codex", "build", "codex.exe")
+        ];
+
+        private string selectedPath = string.Empty;
+
+        public CodexExePath()
+        {
+            foreach (var p in paths)
+            {
+                if (File.Exists(p))
+                {
+                    selectedPath = p;
+                    return;
+                }
+            }
+        }
+
+        public string Get()
+        {
+            return selectedPath;
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexInstanceContainerExtension.cs

@@ -0,0 +1,46 @@
+using CodexClient;
+using KubernetesWorkflow.Types;
+using Utils;
+
+namespace CodexPlugin
+{
+    public static class CodexInstanceContainerExtension
+    {
+        public static ICodexInstance CreateFromPod(RunningPod pod)
+        {
+            var container = pod.Containers.Single();
+
+            return new CodexInstance(
+                name: container.Name,
+                imageName: container.Recipe.Image,
+                startUtc: container.Recipe.RecipeCreatedUtc,
+                discoveryEndpoint: SetClusterInternalIpAddress(pod, container.GetInternalAddress(CodexContainerRecipe.DiscoveryPortTag)),
+                apiEndpoint: container.GetAddress(CodexContainerRecipe.ApiPortTag),
+                listenEndpoint: container.GetInternalAddress(CodexContainerRecipe.ListenPortTag),
+                ethAccount: container.Recipe.Additionals.Get<EthAccount>(),
+                metricsEndpoint: GetMetricsEndpoint(container)
+            );
+        }
+
+        private static Address SetClusterInternalIpAddress(RunningPod pod, Address address)
+        {
+            return new Address(
+                logName: address.LogName,
+                host: pod.PodInfo.Ip,
+                port: address.Port
+            );
+        }
+
+        private static Address? GetMetricsEndpoint(RunningContainer container)
+        {
+            try
+            {
+                return container.GetInternalAddress(CodexContainerRecipe.MetricsPortTag);
+            }
+            catch
+            {
+                return null;
+            }
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -1,317 +0,0 @@
-using CodexPlugin.Hooks;
-using Core;
-using FileUtils;
-using GethPlugin;
-using KubernetesWorkflow;
-using KubernetesWorkflow.Types;
-using Logging;
-using MetricsPlugin;
-using Utils;
-
-namespace CodexPlugin
-{
-    public interface ICodexNode : IHasContainer, IHasMetricsScrapeTarget, IHasEthAddress
-    {
-        string GetName();
-        string GetPeerId();
-        DebugInfo GetDebugInfo();
-        DebugPeer GetDebugPeer(string peerId);
-        ContentId UploadFile(TrackedFile file);
-        ContentId UploadFile(TrackedFile file, Action<Failure> onFailure);
-        ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure);
-        TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
-        TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "");
-        LocalDatasetList LocalFiles();
-        CodexSpace Space();
-        void ConnectToPeer(ICodexNode node);
-        DebugInfoVersion Version { get; }
-        IMarketplaceAccess Marketplace { get; }
-        CrashWatcher CrashWatcher { get; }
-        PodInfo GetPodInfo();
-        ITransferSpeeds TransferSpeeds { get; }
-        EthAccount EthAccount { get; }
-
-        /// <summary>
-        /// Warning! The node is not usable after this.
-        /// TODO: Replace with delete-blocks debug call once available in Codex.
-        /// </summary>
-        void DeleteRepoFolder();
-        void Stop(bool waitTillStopped);
-    }
-
-    public class CodexNode : ICodexNode
-    {
-        private const string UploadFailedMessage = "Unable to store block";
-        private readonly ILog log;
-        private readonly IPluginTools tools;
-        private readonly ICodexNodeHooks hooks;
-        private readonly EthAccount? ethAccount;
-        private readonly TransferSpeeds transferSpeeds;
-        private string peerId = string.Empty;
-        private string nodeId = string.Empty;
-
-        public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, ICodexNodeHooks hooks, EthAccount? ethAccount)
-        {
-            this.tools = tools;
-            this.ethAccount = ethAccount;
-            CodexAccess = codexAccess;
-            Group = group;
-            Marketplace = marketplaceAccess;
-            this.hooks = hooks;
-            Version = new DebugInfoVersion();
-            transferSpeeds = new TransferSpeeds();
-
-            log = new LogPrefixer(tools.GetLog(), $"{GetName()} ");
-        }
-
-        public void Awake()
-        {
-            hooks.OnNodeStarting(Container.Recipe.RecipeCreatedUtc, Container.Recipe.Image, ethAccount);
-        }
-
-        public void Initialize()
-        {
-            hooks.OnNodeStarted(peerId, nodeId);
-        }
-
-        public RunningPod Pod { get { return CodexAccess.Container; } }
-        
-        public RunningContainer Container { get { return Pod.Containers.Single(); } }
-        public CodexAccess CodexAccess { get; }
-        public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; }
-        public CodexNodeGroup Group { get; }
-        public IMarketplaceAccess Marketplace { get; }
-        public DebugInfoVersion Version { get; private set; }
-        public ITransferSpeeds TransferSpeeds { get => transferSpeeds; }
-
-        public IMetricsScrapeTarget MetricsScrapeTarget
-        {
-            get
-            {
-                return new MetricsScrapeTarget(CodexAccess.Container.Containers.First(), CodexContainerRecipe.MetricsPortTag);
-            }
-        }
-
-        public EthAddress EthAddress 
-        {
-            get
-            {
-                EnsureMarketplace();
-                return ethAccount!.EthAddress;
-            }
-        }
-
-        public EthAccount EthAccount
-        {
-            get
-            {
-                EnsureMarketplace();
-                return ethAccount!;
-            }
-        }
-
-        public string GetName()
-        {
-            return Container.Name;
-        }
-
-        public string GetPeerId()
-        {
-            return peerId;
-        }
-
-        public DebugInfo GetDebugInfo()
-        {
-            var debugInfo = CodexAccess.GetDebugInfo();
-            var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
-            Log($"Got DebugInfo with id: {debugInfo.Id}. This node knows: [{known}]");
-            return debugInfo;
-        }
-
-        public DebugPeer GetDebugPeer(string peerId)
-        {
-            return CodexAccess.GetDebugPeer(peerId);
-        }
-
-        public ContentId UploadFile(TrackedFile file)
-        {
-            return UploadFile(file, DoNothing);
-        }
-
-        public ContentId UploadFile(TrackedFile file, Action<Failure> onFailure)
-        {
-            return UploadFile(file, "application/octet-stream", $"attachment; filename=\"{Path.GetFileName(file.Filename)}\"", onFailure);
-        }
-
-        public ContentId UploadFile(TrackedFile file, string contentType, string contentDisposition, Action<Failure> onFailure)
-        {
-            using var fileStream = File.OpenRead(file.Filename);
-            var uniqueId = Guid.NewGuid().ToString();
-            var size = file.GetFilesize();
-
-            hooks.OnFileUploading(uniqueId, size);
-
-            var input = new UploadInput(contentType, contentDisposition, fileStream);
-            var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'...";
-            var measurement = Stopwatch.Measure(log, logMessage, () =>
-            {
-                return CodexAccess.UploadFile(input, onFailure);
-            });
-
-            var response = measurement.Value;
-            transferSpeeds.AddUploadSample(size, measurement.Duration);
-
-            if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
-            if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
-
-            Log($"Uploaded file {file.Describe()}. Received contentId: '{response}'.");
-
-            var cid = new ContentId(response);
-            hooks.OnFileUploaded(uniqueId, size, cid);
-            return cid;
-        }
-
-        public TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "")
-        {
-            return DownloadContent(contentId, DoNothing, fileLabel);
-        }
-
-        public TrackedFile? DownloadContent(ContentId contentId, Action<Failure> onFailure, string fileLabel = "")
-        {
-            var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
-            hooks.OnFileDownloading(contentId);
-            Log($"Downloading '{contentId}'...");
-
-            var logMessage = $"Downloaded '{contentId}' to '{file.Filename}'";
-            var measurement = Stopwatch.Measure(log, logMessage, () => DownloadToFile(contentId.Id, file, onFailure));
-
-            var size = file.GetFilesize();
-            transferSpeeds.AddDownloadSample(size, measurement);
-            hooks.OnFileDownloaded(size, contentId);
-
-            return file;
-        }
-
-        public LocalDatasetList LocalFiles()
-        {
-            return CodexAccess.LocalFiles();
-        }
-
-        public CodexSpace Space()
-        {
-            return CodexAccess.Space();
-        }
-
-        public void ConnectToPeer(ICodexNode node)
-        {
-            var peer = (CodexNode)node;
-
-            Log($"Connecting to peer {peer.GetName()}...");
-            var peerInfo = node.GetDebugInfo();
-            CodexAccess.ConnectToPeer(peerInfo.Id, GetPeerMultiAddresses(peer, peerInfo));
-
-            Log($"Successfully connected to peer {peer.GetName()}.");
-        }
-
-        public PodInfo GetPodInfo()
-        {
-            return CodexAccess.GetPodInfo();
-        }
-
-        public void DeleteRepoFolder()
-        {
-            CodexAccess.DeleteRepoFolder();
-        }
-
-        public void Stop(bool waitTillStopped)
-        {
-            Log("Stopping...");
-            hooks.OnNodeStopping();
-
-            CrashWatcher.Stop();
-            Group.Stop(this, waitTillStopped);
-        }
-
-        public void EnsureOnlineGetVersionResponse()
-        {
-            var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
-            peerId = debugInfo.Id;
-            nodeId = debugInfo.Table.LocalNode.NodeId;
-            var nodeName = CodexAccess.Container.Name;
-
-            if (!debugInfo.Version.IsValid())
-            {
-                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
-            }
-
-            log.AddStringReplace(peerId, nodeName);
-            log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName);
-            log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName);
-            log.AddStringReplace(CodexUtils.ToShortId(debugInfo.Table.LocalNode.NodeId), nodeName);
-            Version = debugInfo.Version;
-        }
-
-        public override string ToString()
-        {
-            return $"CodexNode:{GetName()}";
-        }
-
-        private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)
-        {
-            // The peer we want to connect is in a different pod.
-            // We must replace the default IP with the pod IP in the multiAddress.
-            var workflow = tools.CreateWorkflow();
-            var podInfo = workflow.GetPodInfo(peer.Pod);
-
-            return peerInfo.Addrs.Select(a => a
-                .Replace("0.0.0.0", podInfo.Ip))
-                .ToArray();
-        }
-
-        private void DownloadToFile(string contentId, TrackedFile file, Action<Failure> onFailure)
-        {
-            using var fileStream = File.OpenWrite(file.Filename);
-            var timeout = tools.TimeSet.HttpCallTimeout();
-            try
-            {
-                // Type of stream generated by openAPI client does not support timeouts.
-                var start = DateTime.UtcNow;
-                var cts = new CancellationTokenSource();
-                var downloadTask = Task.Run(() =>
-                {
-                    using var downloadStream = CodexAccess.DownloadFile(contentId, onFailure);
-                    downloadStream.CopyTo(fileStream);
-                }, cts.Token);
-                
-                while (DateTime.UtcNow - start < timeout)
-                {
-                    if (downloadTask.IsFaulted) throw downloadTask.Exception;
-                    if (downloadTask.IsCompletedSuccessfully) return;
-                    Thread.Sleep(100);
-                }
-
-                cts.Cancel();
-                throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
-            }
-            catch
-            {
-                Log($"Failed to download file '{contentId}'.");
-                throw;
-            }
-        }
-
-        private void EnsureMarketplace()
-        {
-            if (ethAccount == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it.");
-        }
-
-        private void Log(string msg)
-        {
-            log.Log(msg);
-        }
-
-        private void DoNothing(Failure failure)
-        {
-        }
-    }
-}

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -1,53 +0,0 @@
-using CodexPlugin.Hooks;
-using Core;
-using GethPlugin;
-using KubernetesWorkflow;
-using KubernetesWorkflow.Types;
-
-namespace CodexPlugin
-{
-    public interface ICodexNodeFactory
-    {
-        CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group);
-        CrashWatcher CreateCrashWatcher(RunningContainer c);
-    }
-
-    public class CodexNodeFactory : ICodexNodeFactory
-    {
-        private readonly IPluginTools tools;
-        private readonly CodexHooksFactory codexHooksFactory;
-
-        public CodexNodeFactory(IPluginTools tools, CodexHooksFactory codexHooksFactory)
-        {
-            this.tools = tools;
-            this.codexHooksFactory = codexHooksFactory;
-        }
-
-        public CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
-        {
-            var ethAccount = GetEthAccount(access);
-            var hooks = codexHooksFactory.CreateHooks(access.Container.Name);
-
-            var marketplaceAccess = GetMarketplaceAccess(access, ethAccount, hooks);
-            return new CodexNode(tools, access, group, marketplaceAccess, hooks, ethAccount);
-        }
-
-        private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAccount? ethAccount, ICodexNodeHooks hooks)
-        {
-            if (ethAccount == null) return new MarketplaceUnavailable();
-            return new MarketplaceAccess(tools.GetLog(), codexAccess, hooks);
-        }
-
-        private EthAccount? GetEthAccount(CodexAccess access)
-        {
-            var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get<EthAccount>();
-            if (ethAccount == null) return null;
-            return ethAccount;
-        }
-
-        public CrashWatcher CreateCrashWatcher(RunningContainer c)
-        {
-            return tools.CreateWorkflow().CreateCrashWatcher(c);
-        }
-    }
-}

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -1,25 +1,23 @@
-using Core;
-using KubernetesWorkflow.Types;
-using MetricsPlugin;
+using CodexClient;
+using Core;
 using System.Collections;
+using Utils;
 
 namespace CodexPlugin
 {
     public interface ICodexNodeGroup : IEnumerable<ICodexNode>, IHasManyMetricScrapeTargets
     {
-        void BringOffline(bool waitTillStopped);
+        void Stop(bool waitTillStopped);
         ICodexNode this[int index] { get; }
     }
 
     public class CodexNodeGroup : ICodexNodeGroup
     {
-        private readonly CodexStarter starter;
+        private readonly ICodexNode[] nodes;
 
-        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningPod[] containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(IPluginTools tools, ICodexNode[] nodes)
         {
-            this.starter = starter;
-            Containers = containers;
-            Nodes = containers.Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
+            this.nodes = nodes;
             Version = new DebugInfoVersion();
         }
 
@@ -31,25 +29,23 @@ namespace CodexPlugin
             }
         }
 
-        public void BringOffline(bool waitTillStopped)
+        public void Stop(bool waitTillStopped)
         {
-            starter.BringOffline(this, waitTillStopped);
-            // Clear everything. Prevent accidental use.
-            Nodes = Array.Empty<CodexNode>();
-            Containers = null!;
+            foreach (var node in Nodes) node.Stop(waitTillStopped);
         }
 
         public void Stop(CodexNode node, bool waitTillStopped)
         {
-            starter.Stop(node.Pod, waitTillStopped);
-            Nodes = Nodes.Where(n => n != node).ToArray();
-            Containers = Containers.Where(c => c != node.Pod).ToArray();
+            node.Stop(waitTillStopped);
         }
 
-        public RunningPod[] Containers { get; private set; }
-        public CodexNode[] Nodes { get; private set; }
+        public ICodexNode[] Nodes => nodes;
         public DebugInfoVersion Version { get; private set; }
-        public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray();
+
+        public Address[] GetMetricsScrapeTargets()
+        {
+            return Nodes.Select(n => n.GetMetricsScrapeTarget()).ToArray();
+        }
 
         public IEnumerator<ICodexNode> GetEnumerator()
         {
@@ -63,12 +59,11 @@ namespace CodexPlugin
 
         public string Describe()
         {
-            return $"group:[{Containers.Describe()}]";
+            return $"group:[{string.Join(",", Nodes.Select(n => n.GetName()))}]";
         }
 
         public void EnsureOnline()
         {
-            foreach (var node in Nodes) node.EnsureOnlineGetVersionResponse();
             var versionResponses = Nodes.Select(n => n.Version);
 
             var first = versionResponses.First();
@@ -79,16 +74,6 @@ namespace CodexPlugin
             }
 
             Version = first;
-            foreach (var node in Nodes) node.Initialize();
-        }
-
-        private CodexNode CreateOnlineCodexNode(RunningPod c, IPluginTools tools, ICodexNodeFactory factory)
-        {
-            var watcher = factory.CreateCrashWatcher(c.Containers.Single());
-            var access = new CodexAccess(tools, c, watcher);
-            var node = factory.CreateOnlineCodexNode(access, this);
-            node.Awake();
-            return node;
         }
     }
 }

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -1,48 +1,68 @@
-using CodexPlugin.Hooks;
+using CodexClient;
+using CodexClient.Hooks;
 using Core;
-using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {
     public class CodexPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata
     {
-        private readonly CodexStarter codexStarter;
+        private const bool UseContainers = true;
+
+        private readonly ICodexStarter codexStarter;
         private readonly IPluginTools tools;
         private readonly CodexLogLevel defaultLogLevel = CodexLogLevel.Trace;
+        private readonly CodexHooksFactory hooksFactory = new CodexHooksFactory();
+        private readonly ProcessControlMap processControlMap = new ProcessControlMap();
+        private readonly CodexWrapper codexWrapper;
 
         public CodexPlugin(IPluginTools tools)
         {
-            codexStarter = new CodexStarter(tools);
             this.tools = tools;
+
+            codexStarter = CreateCodexStarter();
+            codexWrapper = new CodexWrapper(tools, processControlMap, hooksFactory);
+        }
+
+        private ICodexStarter CreateCodexStarter()
+        {
+            if (UseContainers)
+            {
+                Log("Using Containerized Codex instances");
+                return new ContainerCodexStarter(tools, processControlMap);
+            }
+
+            Log("Using Binary Codex instances");
+            return new BinaryCodexStarter(tools, processControlMap);
         }
 
         public string LogPrefix => "(Codex) ";
 
         public void Announce()
         {
-            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
+            Log($"Loaded with Codex ID: '{codexWrapper.GetCodexId()}' - Revision: {codexWrapper.GetCodexRevision()}");
         }
 
         public void AddMetadata(IAddMetadata metadata)
         {
-            metadata.Add("codexid", codexStarter.GetCodexId());
-            metadata.Add("codexrevision", codexStarter.GetCodexRevision());
+            metadata.Add("codexid", codexWrapper.GetCodexId());
+            metadata.Add("codexrevision", codexWrapper.GetCodexRevision());
         }
 
         public void Decommission()
         {
+            codexStarter.Decommission();
         }
 
-        public RunningPod[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        public ICodexInstance[] DeployCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
         {
             var codexSetup = GetSetup(numberOfNodes, setup);
             return codexStarter.BringOnline(codexSetup);
         }
 
-        public ICodexNodeGroup WrapCodexContainers(CoreInterface coreInterface, RunningPod[] containers)
+        public ICodexNodeGroup WrapCodexContainers(ICodexInstance[] instances)
         {
-            containers = containers.Select(c => SerializeGate.Gate(c)).ToArray();
-            return codexStarter.WrapCodexContainers(coreInterface, containers);
+            instances = instances.Select(c => SerializeGate.Gate(c as CodexInstance)).ToArray();
+            return codexWrapper.WrapCodexInstances(instances);
         }
 
         public void WireUpMarketplace(ICodexNodeGroup result, Action<ICodexSetup> setup)
@@ -55,12 +75,17 @@ namespace CodexPlugin
             {
                 mconfig.GethNode.SendEth(node, mconfig.MarketplaceSetup.InitialEth);
                 mconfig.CodexContracts.MintTestTokens(node, mconfig.MarketplaceSetup.InitialTestTokens);
+
+                Log($"Send {mconfig.MarketplaceSetup.InitialEth} and " +
+                    $"minted {mconfig.MarketplaceSetup.InitialTestTokens} for " +
+                    $"{node.GetName()} (address: {node.EthAddress})");
             }
         }
 
-        public void SetCodexHooksProvider(ICodexHooksProvider hooksProvider)
+        public void AddCodexHooksProvider(ICodexHooksProvider hooksProvider)
         {
-            codexStarter.HooksFactory.Provider = hooksProvider;
+            if (hooksFactory.Providers.Contains(hooksProvider)) return;
+            hooksFactory.Providers.Add(hooksProvider);
         }
 
         private CodexSetup GetSetup(int numberOfNodes, Action<ICodexSetup> setup)
@@ -70,5 +95,10 @@ namespace CodexPlugin
             setup(codexSetup);
             return codexSetup;
         }
+
+        private void Log(string msg)
+        {
+            tools.GetLog().Log(msg);
+        }
     }
 }

ProjectPlugins/CodexPlugin/CodexPlugin.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
@@ -6,14 +6,6 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
-  <ItemGroup>
-    <None Remove="openapi.yaml" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <OpenApiReference Include="openapi.yaml" CodeGenerator="NSwagCSharp" Namespace="CodexOpenApi" ClassName="CodexApi" />
-  </ItemGroup>
-
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.ApiDescription.Client" Version="7.0.2">
       <PrivateAssets>all</PrivateAssets>
@@ -30,6 +22,7 @@
     <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
     <ProjectReference Include="..\..\Framework\KubernetesWorkflow\KubernetesWorkflow.csproj" />
     <ProjectReference Include="..\..\Framework\OverwatchTranscript\OverwatchTranscript.csproj" />
+    <ProjectReference Include="..\CodexClient\CodexClient.csproj" />
     <ProjectReference Include="..\CodexContractsPlugin\CodexContractsPlugin.csproj" />
     <ProjectReference Include="..\GethPlugin\GethPlugin.csproj" />
     <ProjectReference Include="..\MetricsPlugin\MetricsPlugin.csproj" />