Overdue readme updates

# Conflicts:
#	ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

.github/workflows/continuous-tests.yaml

@@ -1,117 +0,0 @@
-name: Run Continuous Tests
-
-
-on:
-  # push:
-  #   branches:
-  #     - master
-  #   tags:
-  #     - 'v*.*.*'
-  #   paths-ignore:
-  #     - '**/*.md'
-  #     - '.gitignore'
-  #     - 'docker/**'
-  #     - '!docker/continuous-tests-job.yaml'
-  workflow_dispatch:
-    inputs:
-      source:
-        description: Repository with tests (current)
-        required: false
-        type: string
-      branch:
-        description: Branch with tests (master)
-        required: false
-        type: string
-      nameprefix:
-        description: Runner name prefix (c-tests-runner)
-        required: false
-        type: string
-      namespace:
-        description: Runner namespace (default)
-        required: false
-        type: string
-      tests_target_duration:
-        description: Runner target duration (172800 = 48h)
-        required: false
-        type: string
-      tests_filter:
-        description: Runner tests filter ("")
-        required: false
-        type: string
-      tests_cleanup:
-        description: Runner tests cleanup
-        type: choice
-        options:
-        - true
-        - false
-        default: true
-      deployment_namespace:
-        description: Deployment namespace (c-tests-$runid)
-        required: false
-        type: string
-
-
-env:
-  BRANCH: ${{ github.ref_name }}
-  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
-  NAMEPREFIX: c-tests-runner
-  NAMESPACE: default
-  DEPLOYMENT_NAMESPACE: c-tests
-  TESTS_TARGET_DURATION: 172800
-  TESTS_FILTER: ""
-  TESTS_CLEANUP: true
-  JOB_MANIFEST: docker/continuous-tests-job.yaml
-  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
-  KUBE_VERSION: v1.28.2
-
-
-jobs:
-  run_tests:
-    name: Run Tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Variables
-        run: |
-          RUNID=$(date +%Y%m%d-%H%M%S)
-          echo "RUNID=${RUNID}" >> $GITHUB_ENV
-          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
-          [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ github.event.inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ github.event.inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ github.event.inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.deployment_namespace }}" ]] && echo "DEPLOYMENT_NAMESPACE=${{ github.event.inputs.deployment_namespace }}" >>"$GITHUB_ENV" || echo "DEPLOYMENT_NAMESPACE=${{ env.DEPLOYMENT_NAMESPACE }}-${RUNID}" >>"$GITHUB_ENV"
-
-      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
-        uses: azure/setup-kubectl@v3
-        with:
-          version: ${{ env.KUBE_VERSION }}
-
-      - name: Kubectl - Kubeconfig
-        run: |
-          mkdir -p "${HOME}"/.kube
-          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
-
-      - name: Kubectl - Create Job
-        run: |
-          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
-
-      - name: Tests Identification
-        run: |
-          echo "----"
-          echo "Repository: ${{ env.SOURCE }}"
-          echo "Branch: ${{ env.BRANCH }}"
-          echo "Runner Pod: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
-          echo "Runner namespace: ${{ env.NAMESPACE }}"
-          echo "----"
-          echo "Tests runid: ${{ env.RUNID }}"
-          echo "Tests namespace: ${{ env.DEPLOYMENT_NAMESPACE }}"
-          echo "Tests duration: ${{ env.TESTS_TARGET_DURATION }}"
-          echo "Tests filter: ${{ env.TESTS_FILTER }}"
-          echo "Tests cleanup: ${{ env.TESTS_CLEANUP }}"
-          echo "----"

.github/workflows/docker-discordbot.yml

@@ -0,0 +1,28 @@
+name: Docker - Discord Bot
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/BiblioTech/**'
+      - '!Tools/BiblioTech/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-discordbot.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/BiblioTech/docker/Dockerfile
+      docker_repo: codexstorage/codex-discordbot
+    secrets: inherit
+

.github/workflows/docker-reusable.yml

@@ -1,4 +1,4 @@
-name: Docker - Reusable
+name: Reusable - Docker
 
 
 on:
@@ -66,26 +66,26 @@ jobs:
       PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Docker - Meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.DOCKER_REPO }}
 
       - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Docker - Build and Push by digest
         id: build
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ${{ env.DOCKER_FILE }}
@@ -101,10 +101,10 @@ jobs:
           touch "/tmp/digests/${digest#sha256:}"
 
       - name: Docker - Upload digest
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: digests
-          path: /tmp/digests/*
+          name: digests-${{ matrix.target.arch }}
+          path: /tmp/digests
           if-no-files-found: error
           retention-days: 1
 
@@ -139,17 +139,18 @@ jobs:
           fi
 
       - name: Docker - Download digests
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: digests
+          pattern: digests-*
+          merge-multiple: true
           path: /tmp/digests
 
       - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Docker - Meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.DOCKER_REPO }}
           flavor: |
@@ -161,7 +162,7 @@ jobs:
             type=sha,enable=${{ env.TAG_SHA }}
 
       - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/docker-rewarder.yml

@@ -0,0 +1,26 @@
+name: Docker - Rewarder Bot
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/TestNetRewarder/**'
+      - '!Tools/TestNetRewarder/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-rewarder.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/TestNetRewarder/docker/Dockerfile
+      docker_repo: codexstorage/codex-rewarderbot
+    secrets: inherit
+

.github/workflows/docker-runner.yml

@@ -1,4 +1,4 @@
-name: Docker
+name: Docker - Runner
 
 
 on:
@@ -10,7 +10,7 @@ on:
     paths:
       - docker/Dockerfile
       - docker/docker-entrypoint.sh
-      - .github/workflows/docker.yml
+      - .github/workflows/docker-runner.yml
       - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 

.github/workflows/report-HoldMyBeerTest.yaml

@@ -0,0 +1,20 @@
+name: Report - HoldMyBeerTest
+
+
+on:
+  # schedule:
+  #   - cron: '30 1 */2 * *'
+  workflow_dispatch:
+
+jobs:
+  run_tests:
+    name: Call runner
+    uses: ./.github/workflows/run-continuous-tests.yaml
+    with:
+      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+      branch: master
+      nameprefix: c-tests-report-holdmybeertest-48h
+      tests_target_duration: 48h
+      tests_filter: HoldMyBeerTest
+      tests_cleanup: true
+    secrets: inherit

.github/workflows/report-PeersTest.yaml

@@ -0,0 +1,20 @@
+name: Report - PeersTest
+
+
+on:
+  # schedule:
+  #   - cron: '30 1 */2 * *'
+  workflow_dispatch:
+
+jobs:
+  run_tests:
+    name: Call runner
+    uses: ./.github/workflows/run-continuous-tests.yaml
+    with:
+      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+      branch: master
+      nameprefix: c-tests-report-peerstest-48h
+      tests_target_duration: 48h
+      tests_filter: PeersTest
+      tests_cleanup: true
+    secrets: inherit

.github/workflows/run-continuous-tests.yaml

@@ -0,0 +1,170 @@
+name: Run Continuous Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
+        required: false
+        type: string
+      nameprefix:
+        description: Resources prefix (c-tests)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (2d)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup
+        required: false
+        type: boolean
+        default: true
+  workflow_call:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
+        required: false
+        type: string
+      nameprefix:
+        description: Resources prefix (c-tests)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (2d)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup
+        required: false
+        type: boolean
+        default: true
+      workflow_source:
+        description: Workflow source
+        required: false
+        type: string
+        default: ''
+
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
+  NAMEPREFIX: c-tests
+  NAMESPACE: default
+  TESTS_TARGET_DURATION: 2d
+  TESTS_FILTER: ""
+  TESTS_CLEANUP: true
+  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.28.2
+
+
+jobs:
+  run_tests:
+    name: Run Continuous Tests ${{ inputs.tests_filter }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
+          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          echo "DEPLOYMENT_NAMESPACE=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}"
+          echo "Runner pod: `kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}'`"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+          echo "Tests runid: ${{ env.RUNID }}"
+          echo "Tests codexdockerimage: ${{ env.CODEXDOCKERIMAGE }}"
+          echo "Tests namespace: ${{ env.DEPLOYMENT_NAMESPACE }}"
+          echo "Tests duration: ${{ env.TESTS_TARGET_DURATION }}"
+          echo "Tests filter: ${{ env.TESTS_FILTER }}"
+          echo "Tests cleanup: ${{ env.TESTS_CLEANUP }}"
+          echo "----"
+
+      - name: Show Runner logs
+        if: false
+        run: |
+          # Variables
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          duration=600
+          namespace="${{ env.NAMESPACE }}"
+          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")
+
+          # Check Pod status
+          WAIT=120
+          SECONDS=0
+          sleep=1
+          while (( SECONDS < WAIT )); do
+            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath="{.status.phase}")
+            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in $phase state - Get the logs"; break; } || { echo "Pod $pod is in $phase state - Retry in $sleep seconds / $((WAIT - SECONDS))"; }
+            sleep $sleep
+          done
+
+          # Get logs
+          timeout $duration \
+            kubectl logs $pod \
+              -n $namespace \
+              -f \
+              --tail=-1 \
+              --timestamps || true

.github/workflows/run-dist-tests.yaml

@@ -2,16 +2,6 @@ name: Run Dist Tests
 
 
 on:
-  # push:
-  #   branches:
-  #     - master
-  #   tags:
-  #     - 'v*.*.*'
-  #   paths-ignore:
-  #     - '**/*.md'
-  #     - '.gitignore'
-  #     - 'docker/**'
-  #     - '!docker/dist-tests-job.yaml'
   workflow_dispatch:
     inputs:
       source:
@@ -37,8 +27,8 @@ on:
 
 
 env:
-  BRANCH: ${{ github.ref_name }}
   SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
   NAMEPREFIX: d-tests-runner
   NAMESPACE: default
   COMMAND: dotnet test Tests/CodexTests
@@ -53,21 +43,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Variables
         run: |
-          [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.command }}" ]] && COMMAND="${{ github.event.inputs.command }}" || COMMAND="${{ env.COMMAND }}"
           echo "COMMAND=$(jq -c 'split(" ")' <<< '"'$COMMAND'"')" >>"$GITHUB_ENV"
           echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
           echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
 
       - name: Kubectl - Install ${{ env.KUBE_VERSION }}
-        uses: azure/setup-kubectl@v3
+        uses: azure/setup-kubectl@v4
         with:
           version: ${{ env.KUBE_VERSION }}
 
@@ -85,7 +75,7 @@ jobs:
           echo "----"
           echo "Repository: ${{ env.SOURCE }}"
           echo "Branch: ${{ env.BRANCH }}"
-          echo "Runner Pod: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
           echo "Runner namespace: ${{ env.NAMESPACE }}"
           echo "----"
           echo "Tests runid: ${{ env.RUNID }}"

CONTRIBUTINGPLUGINS.MD

@@ -1,10 +1,26 @@
 # Distributed System Tests for Nim-Codex
 
 ## Contributing plugins
-The testing framework was created for testing Codex. However, it's been designed such that other distributed/containerized projects can 'easily' be added. In order to add your project to the framework you must:
+The testing framework was created for testing Codex. However, it's been designed such that other containerized projects can 'easily' be added.
+
+In this file, you'll see 'users' (in quote) mentioned once or twice. This refers to code/projects/tests which end up making use of your plugin. 'Users' come in many shapes and sizes and tend to have many differen use-cases in mind. Please consider this when reading this document and writing your plugin.
+
+## Checklist
+Your application must pass this checklist to be compatible with the framework:
+- It runs in a docker container.
+- It can be configured via environment variables. (You may need to create a docker image which contains a shell script, to pass some env-vars as CLI arguments to your application. Container command overrides do work, but are not equally reliable across container platforms. When in doubt: use env-var!)
+- It has network interaction:
+   - It exposes one or more APIs via one or more ports, OR
+   - It makes calls to other services. (OR both.)
+
+If your application's use-cases rely primarily on shell interaction, this framework might not be for you. The framework allows you to execute commands in containers AND read stdout/stderr responses. However, its focus during development has always been webservice API interactions.
+
+## Steps
+In order to add your project to the framework you must:
 1. Create a library assembly in the project plugins folder.
 1. It must contain a type that implements the `IProjectPlugin` interface from the `Core` assembly.
-1. If your plugin wants to expose any specific methods or objects to the code using the framework (the tests and tools), it must implement extensions for the `CoreInterface` type.
+1. If your plugin wants to expose any specific methods or objects to 'users', it must implement extensions for the `CoreInterface` type.
+1. If your plugin wants to run containers of its own project, it must provide a recipe.
 
 ## Constructors & Tools
 Your implementation of `IProjectPlugin` must have a public constructor with a single argument of type `IPluginTools`, for example:
@@ -20,19 +36,34 @@ Your implementation of `IProjectPlugin` must have a public constructor with a si
    }
 ```
 
-`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients for containers. (Without having to figure out addresses manually.)
-
 ## Plugin Interfaces
 The `IProjectPlugin` interface requires the implementation of two methods.
-1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version information at this time if applicable.
-1. `Decommission` - Should your plugin have any active system resources, free them in this method.
+1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version and/or configuration information at this time if applicable.
+1. `Decommission` - Should your plugin have any active system resources, free them in this method. Please note that resources managed by the framework (such as running containers and tracked data files) do *not* need to be manually disposed in this method. `Decommission` is to be used for resources not managed by the framework.
 
 There are a few optional interfaces your plugin may choose to implement. The framework will automatically use these interfaces.
-1. `IHasLogPrefix` - Implementing this interface allows you to provide a string with will be prepended to all log statements made by your plugin.
-1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by code that uses your plugin.
+1. `IHasLogPrefix` - Implementing this interface allows you to provide a string which will be prepended to all log statements made by your plugin. A polite thing to do.
+1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by 'users' of your plugin. Often this data finds its way into log files and container-descriptors in order to help track versions/tests/deployments, etc.
+
+## IPluginTools
+`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients to make calls to containers. (Figure out addresses and ports for containers is handled by the framework.)
+
+It is possible and allowed for your plugin to depend on and use other plugins. (For example, maybe your project wants to interact with Ethereum and wants to use the GethPlugin to talk to a Geth node.) `IPluginTools` is *not* what is used for accessing functionality of other plugins. See 'Core Interface' section.
+
+ILog GetLog();
+IHttp CreateHttp(Action<HttpClient> onClientCreated);
+IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
+IHttp CreateHttp();
+IFileManager GetFileManager();
+
+The plugin tools provide:
+1. `Workflow` - This tool allows you to start and stop containers using "container recipes". (More on those below.) It also allows you to execute commands inside a container, access stdout/stderr, detect crashes, and access pod deployment information. The workflow tool also lets you inspect the locations available in the cluster, and decide where you want to run containers. (More on that below as well.)
+1. `Log` - Good logging is priceless. Use this tool to get a log object handle, and write useful debug/info/error statements.
+1. `Http` - This tool gives you a convenient way to access a standard dotnet HttpClient, and takes care of timeouts and retries (in accordance with the config). Additionally, it combos nicely with container objects created by `Workflow`, such that you never have to spend any time figuring out the addresses and ports of your containers.
+1. `FileManager` - Lets you use tracked temporary files. Even if the 'user' tests/application start crashing, the framework will make sure these are cleaned up.
 
 ## Core Interface
-Any functionality your plugin wants to expose to code which uses the framework will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type:
+Any functionality your plugin wants to expose to 'users' will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type:
 ```C#
    public static class CoreInterfaceExtensions
    {
@@ -48,12 +79,14 @@ Any functionality your plugin wants to expose to code which uses the framework w
    }
 ```
 
+If your plugin wants to access the functionality exposed by other plugins, then you can pass the argument `CoreInterface ci` to your plugin code in order to do so. (For example, if you want to start a Geth node, the Geth plugin adds `IGethNode StartGethNode(this CoreInterface ci, Action<IGethSetup> setup)` to the core interface.) Don't forget you'll need to add a project reference to each plugin project you wish to use.
+
 While technically you can build whatever you like on top of the `CoreInterface` and your own plugin types, I recommend that you follow the approach explained below.
 
 ## Deploying, Wrapping, and Starting
 When building a plugin, it is important to make as few assumptions as possible about how it will be used by whoever is going to use the framework. For this reason, I recommend you expose three kinds of methods using your `CoreInterface` extensions:
-1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning containers as a result. If your project requires additional information, you can create a new class type to contain both it and the containers created.
-1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain.
+1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning container objects as a result. If your project requires additional information, you can create a new class type to contain both it and the container objects created.
+1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain. (For example, if my project has a REST API call that allows users to fetch some state information, the object returned by Wrap should have a convenient method to call that API and receive that state information.)
 1. Start - This kind of method does both, simply calling a Deploy method first, then a Wrap method, and returns the result.
 
 Here's an example:
@@ -69,8 +102,8 @@ public static class CoreInterfaceExtensions
 
       public static IMyProjectNode WrapMyProjectContainer(this CoreInterface ci, RunningContainers container)
       {
-         return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` tool to create an HTTP client for the container, then wrap it in an object that
-         // represents the API of your project.
+         return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` to create an HTTP client for the container, then wrap it in an object that
+         // represents the API of your project, in this case 'IMyProjectNode'.
       }
 
       public static IMyProjectNode StartMyProject(this CoreInterface ci, string someArgument)
@@ -82,5 +115,44 @@ public static class CoreInterfaceExtensions
    }
 ```
 
-The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing.
+Should your deploy methods not return framework-types like RunningContainers, please make sure that your custom times are serializable. (Decorate them with the `[Serializable]` attribute.) Tools have been built using this framework which rely on the ability to serialize and store deployment information for later use. Please don't break this possibility. (Consider using the `SerializeGate` type to help ensure compatibility.)
+
+The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinuousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing.
+
+## Container Recipes
+In order to run a container of your application, the framework needs to know how to create that container. Think of a container recipe as being similar to a docker-compose.yaml file: You specify the docker image, ports, environment variables, persistent volumes, and secrets. However, container recipes are code. This allows you to add conditional behaviour to how your container is constructed. For example: The 'user' of your plugin specifies in their call input that they want to run your application in a certain mode. This would cause your container recipe to set certain environment variables, which cause the application to behave in the requested way.
+
+### Addresses and ports
+In a docker-compose.yaml file, it is perfectly normal to specify which ports should be exposed on your container. However, in the framework there's more to consider. When your application container starts, who knows on what kind of machine it runs, and what other processes it's sharing space with? Well, Kubernetes knows. Therefore, it is recommended that container recipes *do not* specify exact port numbers. The framework allows container recipes to declare "a port" without specifying its port number. This allows the framework and Kubernetes to figure out which ports are available when it's time to deploy. In order to find out which port numbers were assigned post-deployment, you can look up the port by tag (which is just an identifying string). When you specify a port to be mapped in your container recipe, you must specify:
+1. `Tag` - An identifier.
+1. `Internal` or `External` - Whether this port should be accessible only inside the cluster (for other containers (k8s: "ClusterIP")) or outside the cluster as well (for external tools/applications (k8s: "NodePort")).
+1. `Protocol` - TCP or UDP. Both protocols on the same port is not universally supported by all container engines, and is therefore not supported by the framework.
+
+If your application wants to listen for incoming traffic from inside its container, be sure to bind it to address "0.0.0.0".
+
+Reminder: If you don't want to worry about addresses, and internal or external ports, you don't have to! The container objects returned by the `workflow` plugin tool have a method called `GetAddress`. Given a port tag, it returns and address object. The `Http` plugin tool can use that address object to set up connections.
+
+## Locations
+The framework is designed to allow you to control instances of your application in multiple (physical) locations. It accomplishes this by using kubernetes, and the ability to deploy containers to specific hosts (nodes) inside a kubernetes cluster. Since Kubernetes allows you to build clusters cross-site, this framework in theory enables you to deploy and interact with containers running anywhere.
+
+The `workflow` plugin tool provides you a list of all available locations in the cluster. When starting a container, you are able to pick one of those locations. If no location is selected, one will be chosen by kubernetes. Locations can be chosen explicitly by kubernetes node name, or, they can be picked from the array of available locations.
+
+Example:
+```C#
+{
+   var location = Ci.GetKnownLocations().Get("kbnode_euwest_paris1");
+   var codex = Ci.StartCodexNode(s => s.At(location));
+}
+```
+In this example, 'Ci' is an instance of the core interface. The CodexPlugin exposes a function 'StartCodexNode', which allows its user to specify a location. This location is then passed to the `workflow` tool when the Codex plugin starts its container.
+
+The available locations array guarantees that each entry corresponds to a different kubernetes host.
+```C#
+{
+   var knownLocations = Ci.GetKnownLocations();
+   // I don't care where exactly, as long as they are different locations.
+   var codexAtZero = Ci.StartCodexNode(s => s.At(knownLocations.Get(0)));
+   var codexAtOne = Ci.StartCodexNode(s => s.At(knownLocations.Get(1)));
+}
+```
 

CONTRIBUTINGTESTS.MD

@@ -21,7 +21,7 @@ Do you want to write some tests using this distributed test setup? Great! Here's
    1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `CodexDistTest` instead.
 1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
 ### Continuous tests
-1. Add new code files to `Tests/CodexContinousTests/Tests`
+1. Add new code files to `Tests/CodexContinuousTests/Tests`
 1. Inherrit from `ContinuousTest`
 1. Define one or more methods and decorate them with the `[TestMoment(...)]` attribute.
 1. The TestMoment takes a number of seconds as argument. Each moment will be executed by the continuous test runner applying the given seconds as delay. (Non-cumulative. So two moments at T:10 will be executed one after another without delay, in this case the order of execution should not be depended upon.)

Framework/Core/DownloadedLog.cs

@@ -4,7 +4,7 @@ namespace Core
 {
     public interface IDownloadedLog
     {
-        bool DoesLogContain(string expectedString);
+        string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
         void DeleteFile();
     }
@@ -18,20 +18,23 @@ namespace Core
             this.logFile = logFile;
         }
 
-        public bool DoesLogContain(string expectedString)
+        public string[] GetLinesContaining(string expectedString)
         {
             using var file = File.OpenRead(logFile.FullFilename);
             using var streamReader = new StreamReader(file);
+            var lines = new List<string>();
 
             var line = streamReader.ReadLine();
             while (line != null)
             {
-                if (line.Contains(expectedString)) return true;
+                if (line.Contains(expectedString))
+                {
+                    lines.Add(line);
+                }
                 line = streamReader.ReadLine();
             }
 
-            //Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
-            return false;
+            return lines.ToArray(); ;
         }
 
         public string[] FindLinesThatContain(params string[] tags)

Framework/Core/Endpoint.cs

@@ -0,0 +1,193 @@
+using Logging;
+using Newtonsoft.Json;
+using Serialization = Newtonsoft.Json.Serialization;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using Utils;
+
+namespace Core
+{
+    public interface IEndpoint
+    {
+        string HttpGetString(string route);
+        T HttpGetJson<T>(string route);
+        TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
+        string HttpPostJson<TRequest>(string route, TRequest body);
+        TResponse HttpPostString<TResponse>(string route, string body);
+        string HttpPostStream(string route, Stream stream);
+        Stream HttpGetStream(string route);
+        T Deserialize<T>(string json);
+    }
+
+    internal class Endpoint : IEndpoint
+    {
+        private readonly ILog log;
+        private readonly IHttp http;
+        private readonly Address address;
+        private readonly string baseUrl;
+        private readonly string? logAlias;
+
+        public Endpoint(ILog log, IHttp http, Address address, string baseUrl, string? logAlias)
+        {
+            this.log = log;
+            this.http = http;
+            this.address = address;
+            this.baseUrl = baseUrl;
+            this.logAlias = logAlias;
+        }
+
+        public string HttpGetString(string route)
+        {
+            return http.OnClient(client =>
+            {
+                return GetString(client, route);
+            }, $"HTTP-GET:{route}");
+        }
+
+        public T HttpGetJson<T>(string route)
+        {
+            return http.OnClient(client =>
+            {
+                var json = GetString(client, route);
+                return Deserialize<T>(json);
+            }, $"HTTP-GET:{route}");
+        }
+
+        public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJson(client, route, body);
+                var json = Time.Wait(response.Content.ReadAsStringAsync());
+                if (!response.IsSuccessStatusCode)
+                {
+                    throw new HttpRequestException(json);
+                }
+                Log(GetUrl() + route, json);
+                return Deserialize<TResponse>(json);
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        public string HttpPostJson<TRequest>(string route, TRequest body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJson(client, route, body);
+                return Time.Wait(response.Content.ReadAsStringAsync());
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        public TResponse HttpPostString<TResponse>(string route, string body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJsonString(client, route, body);
+                if (response == null) throw new Exception("Received no response.");
+                var result = Deserialize<TResponse>(response);
+                if (result == null) throw new Exception("Failed to deserialize response");
+                return result;
+            }, $"HTTO-POST-JSON: {route}");
+        }
+
+        public string HttpPostStream(string route, Stream stream)
+        {
+            return http.OnClient(client =>
+            {
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                var content = new StreamContent(stream);
+                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
+                var response = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(response.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STREAM: {route}");
+        }
+
+        public Stream HttpGetStream(string route)
+        {
+            return http.OnClient(client =>
+            {
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                return Time.Wait(client.GetStreamAsync(url));
+            }, $"HTTP-GET-STREAM: {route}");
+        }
+
+        public T Deserialize<T>(string json)
+        {
+            var errors = new List<string>();
+            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
+            {
+                Error = delegate (object? sender, Serialization.ErrorEventArgs args)
+                {
+                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
+                    {
+                        errors.Add($"""
+                                    Member: '{args.ErrorContext.Member?.ToString() ?? "<null>"}'
+                                    Path: {args.ErrorContext.Path}
+                                    Error: {args.ErrorContext.Error.Message}
+                                    """);
+                        args.ErrorContext.Handled = true;
+                    }
+                }
+            });
+            if (errors.Count > 0)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
+            }
+            else if (deserialized == null)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
+            }
+            return deserialized;
+        }
+
+        private string GetString(HttpClient client, string route)
+        {
+            var url = GetUrl() + route;
+            Log(url, "");
+            var result = Time.Wait(client.GetAsync(url));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
+        private HttpResponseMessage PostJson<TRequest>(HttpClient client, string route, TRequest body)
+        {
+            var url = GetUrl() + route;
+            using var content = JsonContent.Create(body);
+            Log(url, JsonConvert.SerializeObject(body));
+            return Time.Wait(client.PostAsync(url, content));
+        }
+
+        private string PostJsonString(HttpClient client, string route, string body)
+        {
+            var url = GetUrl() + route;
+            Log(url, body);
+            var content = new StringContent(body);
+            content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+            var result = Time.Wait(client.PostAsync(url, content));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
+        private string GetUrl()
+        {
+            return $"{address.Host}:{address.Port}{baseUrl}";
+        }
+
+        private void Log(string url, string message)
+        {
+            if (logAlias != null)
+            {
+                log.Debug($"({logAlias})({url}) = '{message}'", 3);
+            }
+            else
+            {
+                log.Debug($"({url}) = '{message}'", 3);
+            }
+        }
+    }
+}

Framework/Core/Http.cs

@@ -1,22 +1,13 @@
 using Logging;
-using Newtonsoft.Json;
-using Serialization = Newtonsoft.Json.Serialization;
-using System.Net.Http.Headers;
-using System.Net.Http.Json;
 using Utils;
 
 namespace Core
 {
     public interface IHttp
     {
-        string HttpGetString(string route);
-        T HttpGetJson<T>(string route);
-        TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
-        string HttpPostJson<TRequest>(string route, TRequest body);
-        TResponse HttpPostString<TResponse>(string route, string body);
-        string HttpPostStream(string route, Stream stream);
-        Stream HttpGetStream(string route);
-        T Deserialize<T>(string json);
+        T OnClient<T>(Func<HttpClient, T> action);
+        T OnClient<T>(Func<HttpClient, T> action, string description);
+        IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null);
     }
 
     internal class Http : IHttp
@@ -24,185 +15,44 @@ namespace Core
         private static readonly object httpLock = new object();
         private readonly ILog log;
         private readonly ITimeSet timeSet;
-        private readonly Address address;
-        private readonly string baseUrl;
         private readonly Action<HttpClient> onClientCreated;
-        private readonly string? logAlias;
 
-        internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
-            : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
+        internal Http(ILog log, ITimeSet timeSet)
+            : this(log, timeSet, DoNothing)
         {
         }
 
-        internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
+        internal Http(ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
         {
             this.log = log;
             this.timeSet = timeSet;
-            this.address = address;
-            this.baseUrl = baseUrl;
             this.onClientCreated = onClientCreated;
-            this.logAlias = logAlias;
-            if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
-            if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
         }
 
-        public string HttpGetString(string route)
+        public T OnClient<T>(Func<HttpClient, T> action)
         {
+            return OnClient(action, GetDescription());
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action, string description)
+        {
+            var client = GetClient();
+
             return LockRetry(() =>
             {
-                return GetString(route);
-            }, $"HTTP-GET:{route}");
+                return action(client);
+            }, description);
         }
 
-        public T HttpGetJson<T>(string route)
+        public IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null)
         {
-            return LockRetry(() =>
-            {
-                var json = GetString(route);
-                return Deserialize<T>(json);
-            }, $"HTTP-GET:{route}");
+            return new Endpoint(log, this, address, baseUrl, logAlias);
         }
 
-        public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
+        private string GetDescription()
         {
-            return LockRetry(() =>
-            {
-                var response = PostJson(route, body);
-                var json = Time.Wait(response.Content.ReadAsStringAsync());
-                if (!response.IsSuccessStatusCode)
-                {
-                    throw new HttpRequestException(json);
-                }
-                Log(GetUrl() + route, json);
-                return Deserialize<TResponse>(json);
-            }, $"HTTP-POST-JSON: {route}");
-        }
-
-        public string HttpPostJson<TRequest>(string route, TRequest body)
-        {
-            return LockRetry(() =>
-            {
-                var response = PostJson(route, body);
-                return Time.Wait(response.Content.ReadAsStringAsync());
-            }, $"HTTP-POST-JSON: {route}");
-        }
-
-        public TResponse HttpPostString<TResponse>(string route, string body)
-        {
-            return LockRetry(() =>
-            {
-                var response = PostJsonString(route, body);
-                if (response == null) throw new Exception("Received no response.");
-                var result = Deserialize<TResponse>(response);
-                if (result == null) throw new Exception("Failed to deserialize response");
-                return result;
-            }, $"HTTO-POST-JSON: {route}");
-        }
-
-        public string HttpPostStream(string route, Stream stream)
-        {
-            return LockRetry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, "~ STREAM ~");
-                var content = new StreamContent(stream);
-                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
-                var response = Time.Wait(client.PostAsync(url, content));
-                var str = Time.Wait(response.Content.ReadAsStringAsync());
-                Log(url, str);
-                return str;
-            }, $"HTTP-POST-STREAM: {route}");
-        }
-
-        public Stream HttpGetStream(string route)
-        {
-            return LockRetry(() =>
-            {
-                var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, "~ STREAM ~");
-                return Time.Wait(client.GetStreamAsync(url));
-            }, $"HTTP-GET-STREAM: {route}");
-        }
-
-        public T Deserialize<T>(string json)
-        {
-            var errors = new List<string>();
-            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
-            {
-                Error = delegate(object? sender, Serialization.ErrorEventArgs args)
-                {
-                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
-                    {
-                        errors.Add($"""
-                                    Member: '{args.ErrorContext.Member?.ToString() ?? "<null>"}'
-                                    Path: {args.ErrorContext.Path}
-                                    Error: {args.ErrorContext.Error.Message}
-                                    """);
-                        args.ErrorContext.Handled = true;
-                    }
-                }
-            });
-            if (errors.Count > 0)
-            {
-                throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
-            }
-            else if (deserialized == null)
-            {
-                throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
-            }
-            return deserialized;
-        }
-
-        private string GetString(string route)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            Log(url, "");
-            var result = Time.Wait(client.GetAsync(url));
-            var str = Time.Wait(result.Content.ReadAsStringAsync());
-            Log(url, str);
-            return str;
-        }
-
-        private HttpResponseMessage PostJson<TRequest>(string route, TRequest body)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            using var content = JsonContent.Create(body);
-            Log(url, JsonConvert.SerializeObject(body));
-            return Time.Wait(client.PostAsync(url, content));
-        }
-
-        private string PostJsonString(string route, string body)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            Log(url, body);
-            var content = new StringContent(body);
-            content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
-            var result = Time.Wait(client.PostAsync(url, content));
-            var str = Time.Wait(result.Content.ReadAsStringAsync());
-            Log(url, str);
-            return str;
-        }
-
-        private string GetUrl()
-        {
-            return $"{address.Host}:{address.Port}{baseUrl}";
-        }
-
-        private void Log(string url, string message)
-        {
-            if (logAlias != null)
-            {
-                log.Debug($"({logAlias})({url}) = '{message}'", 3);
-            }
-            else
-            {
-                log.Debug($"({url}) = '{message}'", 3);
-            }
+            // todo: check this:
+            return DebugStack.GetCallerName(skipFrames: 2);
         }
 
         private T LockRetry<T>(Func<T> operation, string description)

Framework/Core/PluginTools.cs

@@ -1,7 +1,6 @@
 using FileUtils;
 using KubernetesWorkflow;
 using Logging;
-using Utils;
 
 namespace Core
 {
@@ -22,9 +21,9 @@ namespace Core
 
     public interface IHttpFactoryTool
     {
-        IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null);
-        IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet timeSet, string? logAlias = null);
-        IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null);
+        IHttp CreateHttp(Action<HttpClient> onClientCreated);
+        IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
+        IHttp CreateHttp();
     }
 
     public interface IFileTool
@@ -37,11 +36,11 @@ namespace Core
         private readonly ITimeSet timeSet;
         private readonly WorkflowCreator workflowCreator;
         private readonly IFileManager fileManager;
-        private ILog log;
+        private readonly LogPrefixer log;
 
         internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet)
         {
-            this.log = log;
+            this.log = new LogPrefixer(log);
             this.workflowCreator = workflowCreator;
             this.timeSet = timeSet;
             fileManager = new FileManager(log, fileManagerRootFolder);
@@ -49,22 +48,22 @@ namespace Core
 
         public void ApplyLogPrefix(string prefix)
         {
-            log = new LogPrefixer(log, prefix);
+            log.Prefix = prefix;
         }
 
-        public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
+        public IHttp CreateHttp(Action<HttpClient> onClientCreated)
         {
-            return CreateHttp(address, baseUrl, onClientCreated, timeSet, logAlias);
+            return CreateHttp(onClientCreated, timeSet);
         }
 
-        public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet ts, string? logAlias = null)
+        public IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet ts)
         {
-            return new Http(log, ts, address, baseUrl, onClientCreated, logAlias);
+            return new Http(log, ts, onClientCreated);
         }
 
-        public IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null)
+        public IHttp CreateHttp()
         {
-            return new Http(log, timeSet, address, baseUrl, logAlias);
+            return new Http(log, timeSet);
         }
 
         public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)

Framework/DiscordRewards/CheckConfig.cs

@@ -0,0 +1,21 @@
+using Utils;
+
+namespace DiscordRewards
+{
+    public class CheckConfig
+    {
+        public CheckType Type { get; set; }
+        public ulong MinNumberOfHosts { get; set; }
+        public ByteSize MinSlotSize { get; set; } = 0.Bytes();
+        public TimeSpan MinDuration { get; set; } = TimeSpan.Zero;
+    }
+
+    public enum CheckType
+    {
+        Uninitialized,
+        FilledSlot,
+        FinishedSlot,
+        PostedContract,
+        StartedContract,
+    }
+}

Framework/DiscordRewards/DiscordRewards.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -0,0 +1,13 @@
+namespace DiscordRewards
+{
+    public class GiveRewardsCommand
+    {
+        public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
+    }
+
+    public class RewardUsersCommand
+    {
+        public ulong RewardId { get; set; }
+        public string[] UserAddresses { get; set; } = Array.Empty<string>();
+    }
+}

Framework/DiscordRewards/RewardConfig.cs

@@ -0,0 +1,18 @@
+namespace DiscordRewards
+{
+    public class RewardConfig
+    {
+        public const string UsernameTag = "<USER>";
+
+        public RewardConfig(ulong roleId, string message, CheckConfig checkConfig)
+        {
+            RoleId = roleId;
+            Message = message;
+            CheckConfig = checkConfig;
+        }
+
+        public ulong RoleId { get; }
+        public string Message { get; }
+        public CheckConfig CheckConfig { get; }
+    }
+}

Framework/DiscordRewards/RewardRepo.cs

@@ -0,0 +1,53 @@
+using Utils;
+
+namespace DiscordRewards
+{
+    public class RewardRepo
+    {
+        private static string Tag => RewardConfig.UsernameTag;
+
+        public RewardConfig[] Rewards { get; } = new RewardConfig[]
+        {
+            // Filled any slot
+            new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
+            {
+                Type = CheckType.FilledSlot
+            }),
+
+            // Finished any slot
+            new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
+            {
+                Type = CheckType.FinishedSlot
+            }),
+
+            // Finished a sizable slot
+            new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot!", new CheckConfig
+            {
+                Type = CheckType.FinishedSlot,
+                MinSlotSize = 1.GB(),
+                MinDuration = TimeSpan.FromHours(24.0),
+            }),
+
+            // Posted any contract
+            new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
+            {
+                Type = CheckType.PostedContract
+            }),
+
+            // Started any contract
+            new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
+            {
+                Type = CheckType.StartedContract
+            }),
+
+            // Started a sizable contract
+            new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time!", new CheckConfig
+            {
+                Type = CheckType.FinishedSlot,
+                MinNumberOfHosts = 4,
+                MinSlotSize = 1.GB(),
+                MinDuration = TimeSpan.FromHours(24.0),
+            })
+        };
+    }
+}

Framework/FileUtils/FileManager.cs

@@ -45,7 +45,7 @@ namespace FileUtils
         {
             var sw = Stopwatch.Begin(log);
             var result = GenerateRandomFile(size, label);
-            sw.End($"Generated file '{result.Describe()}'.");
+            sw.End($"Generated file {result.Describe()}.");
             return result;
         }
 

Framework/FileUtils/TrackedFile.cs

@@ -37,6 +37,11 @@ namespace FileUtils
             return $"'{Filename}'{sizePostfix}";
         }
 
+        public ByteSize GetFilesize()
+        {
+            return new ByteSize(GetFileSize());
+        }
+
         private void AssertEqual(TrackedFile? actual)
         {
             if (actual == null)  FrameworkAssert.Fail("TestFile is null.");

Framework/GethConnector/GethConnector.cs

@@ -0,0 +1,39 @@
+using CodexContractsPlugin;
+using GethPlugin;
+using Logging;
+
+namespace GethConnector
+{
+    public class GethConnector
+    {
+        public IGethNode GethNode { get; }
+        public ICodexContracts CodexContracts { get; }
+
+        public static GethConnector? Initialize(ILog log)
+        {
+            if (!string.IsNullOrEmpty(GethInput.LoadError))
+            {
+                var msg = "Geth input incorrect: " + GethInput.LoadError;
+                log.Error(msg);
+                return null;
+            }
+
+            var contractsDeployment = new CodexContractsDeployment(
+                marketplaceAddress: GethInput.MarketplaceAddress,
+                abi: GethInput.ABI,
+                tokenAddress: GethInput.TokenAddress
+            );
+
+            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+            var contracts = new CodexContractsAccess(log, gethNode, contractsDeployment);
+
+            return new GethConnector(gethNode, contracts);
+        }
+
+        private GethConnector(IGethNode gethNode, ICodexContracts codexContracts)
+        {
+            GethNode = gethNode;
+            CodexContracts = codexContracts;
+        }
+    }
+}

Framework/GethConnector/GethConnector.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\GethPlugin\GethPlugin.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/GethConnector/GethInput.cs

@@ -0,0 +1,52 @@
+namespace GethConnector
+{
+    public static class GethInput
+    {
+        private const string GethHostVar = "GETH_HOST";
+        private const string GethPortVar = "GETH_HTTP_PORT";
+        private const string GethPrivKeyVar = "GETH_PRIVATE_KEY";
+        private const string MarketplaceAddressVar = "CODEXCONTRACTS_MARKETPLACEADDRESS";
+        private const string TokenAddressVar = "CODEXCONTRACTS_TOKENADDRESS";
+        private const string AbiVar = "CODEXCONTRACTS_ABI";
+
+        static GethInput()
+        {
+            var error = new List<string>();
+            var gethHost = GetEnvVar(error, GethHostVar);
+            var gethPort = Convert.ToInt32(GetEnvVar(error, GethPortVar));
+            var privateKey = GetEnvVar(error, GethPrivKeyVar);
+            var marketplaceAddress = GetEnvVar(error, MarketplaceAddressVar);
+            var tokenAddress = GetEnvVar(error, TokenAddressVar);
+            var abi = GetEnvVar(error, AbiVar);
+
+            if (error.Any())
+            {
+                LoadError = string.Join(", ", error);
+            }
+            else
+            {
+                GethHost = gethHost!;
+                GethPort = gethPort;
+                PrivateKey = privateKey!;
+                MarketplaceAddress = marketplaceAddress!;
+                TokenAddress = tokenAddress!;
+                ABI = abi!;
+            }
+        }
+
+        public static string GethHost { get; } = string.Empty;
+        public static int GethPort { get; }
+        public static string PrivateKey { get; } = string.Empty;
+        public static string MarketplaceAddress { get; } = string.Empty;
+        public static string TokenAddress { get; } = string.Empty;
+        public static string ABI { get; } = string.Empty;
+        public static string LoadError { get; } = string.Empty;
+
+        private static string? GetEnvVar(List<string> error, string name)
+        {
+            var result = Environment.GetEnvironmentVariable(name);
+            if (string.IsNullOrEmpty(result)) error.Add($"'{name}' is not set.");
+            return result;
+        }
+    }
+}

Framework/KubernetesWorkflow/K8sController.cs

@@ -49,13 +49,14 @@ namespace KubernetesWorkflow
             return CreatePodInfo(pod);
         }
 
-        public void Stop(StartResult startResult)
+        public void Stop(StartResult startResult, bool waitTillStopped)
         {
             log.Debug();
             if (startResult.InternalService != null) DeleteService(startResult.InternalService);
             if (startResult.ExternalService != null) DeleteService(startResult.ExternalService);
             DeleteDeployment(startResult.Deployment);
-            WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
+
+            if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
         }
 
         public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
@@ -84,6 +85,31 @@ namespace KubernetesWorkflow
             return result;
         }
 
+        public int[] GetUsedExternalPorts()
+        {
+            return client.Run(c =>
+            {
+                var result = new List<int>();
+
+                var services = c.ListServiceForAllNamespaces();
+                var nodePorts = services.Items.Where(s => s.Spec.Type == "NodePort").ToArray();
+                if (!nodePorts.Any()) return result.ToArray();
+
+                foreach (var service in nodePorts)
+                {
+                    foreach (var port in service.Spec.Ports)
+                    {
+                        if (port.NodePort.HasValue)
+                        {
+                            result.Add(port.NodePort.Value);
+                        }
+                    }
+                }
+
+                return result.ToArray();
+            });
+        }
+
         public void DeleteAllNamespacesStartingWith(string prefix)
         {
             log.Debug();
@@ -335,6 +361,8 @@ namespace KubernetesWorkflow
                         },
                         Spec = new V1PodSpec
                         {
+                            PriorityClassName = GetPriorityClassName(containerRecipes),
+                            Affinity = CreatePodAffinity(containerRecipes),
                             NodeSelector = CreateNodeSelector(location),
                             Containers = CreateDeploymentContainers(containerRecipes),
                             Volumes = CreateVolumes(containerRecipes)
@@ -367,12 +395,57 @@ namespace KubernetesWorkflow
             };
         }
 
+        private V1Affinity? CreatePodAffinity(ContainerRecipe[] recipes)
+        {
+            var notIns = recipes
+                .Select(r => r.SchedulingAffinity.NotIn)
+                .Where(n => !string.IsNullOrEmpty(n))
+                .Distinct()
+                .ToList();
+
+            if (!notIns.Any()) return null;
+
+            return new V1Affinity
+            {
+                NodeAffinity = new V1NodeAffinity
+                {
+                    RequiredDuringSchedulingIgnoredDuringExecution = new V1NodeSelector
+                    {
+                        NodeSelectorTerms = new List<V1NodeSelectorTerm>
+                        { 
+                            new V1NodeSelectorTerm
+                            {
+                                MatchExpressions = new List<V1NodeSelectorRequirement>
+                                {
+                                    new V1NodeSelectorRequirement
+                                    {
+                                        Key = "allow-tests-pods",
+                                        OperatorProperty = "NotIn",
+                                        Values = notIns
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            };
+        }
+
         private K8sNodeLabel? GetNodeLabelForLocation(ILocation location)
         {
             var l = (Location)location;
             return l.NodeLabel;
         }
 
+        private string GetPriorityClassName(ContainerRecipe[] containerRecipes)
+        {
+            if (containerRecipes.Any(c => c.SetCriticalPriority))
+            {
+                return "system-node-critical";
+            }
+            return null!;
+        }
+
         private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes)
         {
             return containerRecipes.First().PodLabels.GetLabels();
@@ -426,10 +499,17 @@ namespace KubernetesWorkflow
                 Ports = CreateContainerPorts(recipe),
                 Env = CreateEnv(recipe),
                 VolumeMounts = CreateContainerVolumeMounts(recipe),
-                Resources = CreateResourceLimits(recipe)
+                Resources = CreateResourceLimits(recipe),
+                Command = CreateCommandList(recipe)
             };
         }
 
+        private IList<string> CreateCommandList(ContainerRecipe recipe)
+        {
+            if (recipe.CommandOverride == null || !recipe.CommandOverride.Command.Any()) return null!;
+            return recipe.CommandOverride.Command.ToList();
+        }
+
         private V1ResourceRequirements CreateResourceLimits(ContainerRecipe recipe)
         {
             return new V1ResourceRequirements
@@ -627,7 +707,8 @@ namespace KubernetesWorkflow
         private V1Pod GetPodForDeployment(RunningDeployment deployment)
         {
             return Time.Retry(() => GetPodForDeplomentInternal(deployment),
-                maxRetries: 2,
+                // We will wait up to 1 minute, k8s might be moving pods around.
+                maxRetries: 6,
                 retryTime: TimeSpan.FromSeconds(10),
                 description: "Find pod by label for deployment.");
         }
@@ -656,17 +737,17 @@ namespace KubernetesWorkflow
 
         private RunningService? CreateInternalService(ContainerRecipe[] recipes)
         {
-            return CreateService(recipes, r => r.InternalPorts.Concat(r.ExposedPorts).ToArray(), "ClusterIP", "int");
+            return CreateService(recipes, r => r.InternalPorts.Concat(r.ExposedPorts).ToArray(), "ClusterIP", "int", false);
         }
 
         private RunningService? CreateExternalService(ContainerRecipe[] recipes)
         {
-            return CreateService(recipes, r => r.ExposedPorts, "NodePort", "ext");
+            return CreateService(recipes, r => r.ExposedPorts, "NodePort", "ext", true);
         }
 
-        private RunningService? CreateService(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector, string serviceType, string namePostfix)
+        private RunningService? CreateService(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector, string serviceType, string namePostfix, bool isNodePort)
         {
-            var ports = CreateServicePorts(recipes, portSelector);
+            var ports = CreateServicePorts(recipes, portSelector, isNodePort);
             if (!ports.Any()) return null;
 
             var serviceSpec = new V1Service
@@ -740,7 +821,7 @@ namespace KubernetesWorkflow
             };
         }
 
-        private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector)
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector, bool isNodePort)
         {
             var result = new List<V1ServicePort>();
             foreach (var recipe in recipes)
@@ -748,29 +829,33 @@ namespace KubernetesWorkflow
                 var ports = portSelector(recipe);
                 foreach (var port in ports)
                 {
-                    result.AddRange(CreateServicePorts(recipe, port));
+                    result.AddRange(CreateServicePorts(recipe, port, isNodePort));
                 }
             }
             return result;
         }
 
-        private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe, Port recipePort)
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe, Port recipePort, bool isNodePort)
         {
             var result = new List<V1ServicePort>();
-            if (recipePort.IsTcp()) CreateServicePort(result, recipe, recipePort, "TCP");
-            if (recipePort.IsUdp()) CreateServicePort(result, recipe, recipePort, "UDP");
+            if (recipePort.IsTcp()) CreateServicePort(result, recipe, recipePort, "TCP", isNodePort);
+            if (recipePort.IsUdp()) CreateServicePort(result, recipe, recipePort, "UDP", isNodePort);
             return result;
         }
 
-        private void CreateServicePort(List<V1ServicePort> result, ContainerRecipe recipe, Port port, string protocol)
+        private void CreateServicePort(List<V1ServicePort> result, ContainerRecipe recipe, Port port, string protocol, bool isNodePort)
         {
-            result.Add(new V1ServicePort
+            var p = new V1ServicePort
             {
                 Name = GetNameForPort(recipe, port),
                 Protocol = protocol,
                 Port = port.Number,
                 TargetPort = GetNameForPort(recipe, port)
-            });
+            };
+
+            if (isNodePort) p.NodePort = port.Number;
+
+            result.Add(p);
         }
 
         #endregion

Framework/KubernetesWorkflow/PublicIpService.cs

@@ -0,0 +1,7 @@
+namespace KubernetesWorkflow
+{
+    public static class PublicIpService
+    {
+        public static string Address { get; } = "ip.codex.storage";
+    }
+}

Framework/KubernetesWorkflow/Recipe/CommandOverride.cs

@@ -0,0 +1,12 @@
+namespace KubernetesWorkflow.Recipe
+{
+    public class CommandOverride
+    {
+        public CommandOverride(params string[] command)
+        {
+            Command = command;
+        }
+
+        public string[] Command { get; }
+    }
+}

Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs

@@ -2,12 +2,15 @@
 {
     public class ContainerRecipe
     {
-        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
+        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, CommandOverride commandOverride, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
         {
             Number = number;
             NameOverride = nameOverride;
             Image = image;
             Resources = resources;
+            SchedulingAffinity = schedulingAffinity;
+            CommandOverride = commandOverride;
+            SetCriticalPriority = setCriticalPriority;
             ExposedPorts = exposedPorts;
             InternalPorts = internalPorts;
             EnvVars = envVars;
@@ -32,6 +35,9 @@
         public int Number { get; }
         public string? NameOverride { get; }
         public ContainerResources Resources { get; }
+        public SchedulingAffinity SchedulingAffinity { get; }
+        public CommandOverride CommandOverride { get; }
+        public bool SetCriticalPriority { get; }
         public string Image { get; }
         public Port[] ExposedPorts { get; }
         public Port[] InternalPorts { get; }
@@ -53,6 +59,7 @@
                 $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
                 $"envVars: {string.Join(",", EnvVars.Select(v => v.ToString()))}, " +
                 $"limits: {Resources}, " +
+                $"affinity: {SchedulingAffinity}, " +
                 $"volumes: {string.Join(",", Volumes.Select(v => $"'{v.MountPath}'"))}";
         }
     }

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -13,6 +13,9 @@ namespace KubernetesWorkflow.Recipe
         private readonly List<object> additionals = new List<object>();
         private RecipeComponentFactory factory = null!;
         private ContainerResources resources = new ContainerResources();
+        private SchedulingAffinity schedulingAffinity = new SchedulingAffinity();
+        private CommandOverride commandOverride = new CommandOverride();
+        private bool setCriticalPriority;
 
         public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config)
         {
@@ -22,7 +25,7 @@ namespace KubernetesWorkflow.Recipe
 
             Initialize(config);
 
-            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources,
+            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, commandOverride, setCriticalPriority,
                 exposedPorts.ToArray(),
                 internalPorts.ToArray(),
                 envVars.ToArray(),
@@ -40,6 +43,9 @@ namespace KubernetesWorkflow.Recipe
             additionals.Clear();
             this.factory = null!;
             resources = new ContainerResources();
+            schedulingAffinity = new SchedulingAffinity();
+            commandOverride = new CommandOverride();
+            setCriticalPriority = false;
 
             return recipe;
         }
@@ -52,17 +58,17 @@ namespace KubernetesWorkflow.Recipe
 
         protected Port AddExposedPort(string tag, PortProtocol protocol = PortProtocol.TCP)
         {
-            return AddExposedPort(factory.CreatePort(tag, protocol));
+            return AddExposedPort(factory.CreateExternalPort(tag, protocol));
         }
 
         protected Port AddExposedPort(int number, string tag, PortProtocol protocol = PortProtocol.TCP)
         {
-            return AddExposedPort(factory.CreatePort(number, tag, protocol));
+            return AddExposedPort(factory.CreateExternalPort(number, tag, protocol));
         }
 
         protected Port AddInternalPort(string tag = "", PortProtocol protocol = PortProtocol.TCP)
         {
-            var p = factory.CreatePort(tag, protocol);
+            var p = factory.CreateInternalPort(tag, protocol);
             internalPorts.Add(p);
             return p;
         }
@@ -121,6 +127,21 @@ namespace KubernetesWorkflow.Recipe
             SetResourcesRequest(new ContainerResourceSet(milliCPUs, memory));
         }
 
+        protected void SetSchedulingAffinity(string notIn)
+        {
+            schedulingAffinity = new SchedulingAffinity(notIn);
+        }
+
+        protected void OverrideCommand(params string[] command)
+        {
+            commandOverride = new CommandOverride(command);
+        }
+
+        protected void SetSystemCriticalPriority()
+        {
+            setCriticalPriority = true;
+        }
+
         // Disabled following a possible bug in the k8s cluster that will throttle containers much more than is
         // called for if they have resource limits defined.
         //protected void SetResourceLimits(int milliCPUs, ByteSize memory)

Framework/KubernetesWorkflow/Recipe/RecipeComponentFactory.cs

@@ -5,16 +5,36 @@ namespace KubernetesWorkflow.Recipe
 {
     public class RecipeComponentFactory
     {
-        private NumberSource portNumberSource = new NumberSource(8080);
+        private readonly NumberSource internalNumberSource = new NumberSource(8080);
+        private static readonly NumberSource externalNumberSource = new NumberSource(30000);
+        private static int[] usedExternalPorts = Array.Empty<int>();
 
-        public Port CreatePort(int number, string tag, PortProtocol protocol)
+        public void Update(K8sController controller)
         {
+            usedExternalPorts = controller.GetUsedExternalPorts();
+        }
+
+        public Port CreateInternalPort(string tag, PortProtocol protocol)
+        {
+            return new Port(internalNumberSource.GetNextNumber(), tag, protocol);
+        }
+
+        public Port CreateExternalPort(int number, string tag, PortProtocol protocol)
+        {
+            if (usedExternalPorts.Contains(number)) throw new Exception($"External port number {number} is already in use by the cluster.");
             return new Port(number, tag, protocol);
         }
 
-        public Port CreatePort(string tag, PortProtocol protocol)
+        public Port CreateExternalPort(string tag, PortProtocol protocol)
         {
-            return new Port(portNumberSource.GetNextNumber(), tag, protocol);
+            while (true)
+            {
+                var number = externalNumberSource.GetNextNumber();
+                if (!usedExternalPorts.Contains(number))
+                {
+                    return new Port(number, tag, protocol);
+                }
+            }
         }
 
         public EnvVar CreateEnvVar(string name, int value)

Framework/KubernetesWorkflow/Recipe/SchedulingAffinity.cs

@@ -0,0 +1,18 @@
+namespace KubernetesWorkflow.Recipe
+{
+    public class SchedulingAffinity
+    {
+        public SchedulingAffinity(string? notIn = null)
+        {
+            NotIn = notIn;
+        }
+
+        public string? NotIn { get; }
+
+        public override string ToString()
+        {
+            if (string.IsNullOrEmpty(NotIn)) return "none";
+            return "notIn:" + NotIn;
+        }
+    }
+}

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -14,7 +14,7 @@ namespace KubernetesWorkflow
         PodInfo GetPodInfo(RunningContainer container);
         PodInfo GetPodInfo(RunningContainers containers);
         CrashWatcher CreateCrashWatcher(RunningContainer container);
-        void Stop(RunningContainers containers);
+        void Stop(RunningContainers containers, bool waitTillStopped);
         void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
         void DeleteNamespace();
@@ -54,12 +54,19 @@ namespace KubernetesWorkflow
         {
             return K8s(controller =>
             {
+                componentFactory.Update(controller);
+
                 var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
                 var rc = new RunningContainers(startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
+
+                if (startResult.ExternalService != null)
+                {
+                    componentFactory.Update(controller);
+                }
                 return rc;
             });
         }
@@ -79,11 +86,11 @@ namespace KubernetesWorkflow
             return K8s(c => c.CreateCrashWatcher(container));
         }
 
-        public void Stop(RunningContainers runningContainers)
+        public void Stop(RunningContainers runningContainers, bool waitTillStopped)
         {
             K8s(controller =>
             {
-                controller.Stop(runningContainers.StartResult);
+                controller.Stop(runningContainers.StartResult, waitTillStopped);
                 cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
             });
         }
@@ -222,7 +229,7 @@ namespace KubernetesWorkflow
             }
             catch (k8s.Autorest.HttpOperationException ex)
             {
-                log.Error(JsonConvert.SerializeObject(ex));
+                log.Error(JsonConvert.SerializeObject(ex.Response));
                 throw;
             }
         }
@@ -238,7 +245,7 @@ namespace KubernetesWorkflow
             }
             catch (k8s.Autorest.HttpOperationException ex)
             {
-                log.Error(JsonConvert.SerializeObject(ex));
+                log.Error(JsonConvert.SerializeObject(ex.Response));
                 throw;
             }
         }

Framework/Logging/BaseLog.cs

@@ -55,7 +55,9 @@ namespace Logging
 
         public virtual void Error(string message)
         {
-            Log($"[ERROR] {message}");
+            var msg = $"[ERROR] {message}";
+            Console.WriteLine(msg);
+            Log(msg);
         }
 
         public virtual void AddStringReplace(string from, string to)

Framework/Logging/FileLog.cs

@@ -0,0 +1,17 @@
+namespace Logging
+{
+    public class FileLog : BaseLog
+    {
+        public FileLog(string fullFilename)
+        {
+            FullFilename = fullFilename;
+        }
+
+        public string FullFilename { get; }
+
+        protected override string GetFullName()
+        {
+            return FullFilename;
+        }
+    }
+}

Framework/Logging/LogPrefixer.cs

@@ -3,14 +3,21 @@
     public class LogPrefixer : ILog
     {
         private readonly ILog backingLog;
-        private readonly string prefix;
+
+        public LogPrefixer(ILog backingLog)
+        {
+            this.backingLog = backingLog;
+        }
 
         public LogPrefixer(ILog backingLog, string prefix)
         {
             this.backingLog = backingLog;
-            this.prefix = prefix;
+            Prefix = prefix;
         }
 
+        public string Prefix { get; set; } = string.Empty;
+
+
         public LogFile CreateSubfile(string ext = "log")
         {
             return backingLog.CreateSubfile(ext);
@@ -18,17 +25,17 @@
 
         public void Debug(string message = "", int skipFrames = 0)
         {
-            backingLog.Debug(prefix + message, skipFrames);
+            backingLog.Debug(Prefix + message, skipFrames);
         }
 
         public void Error(string message)
         {
-            backingLog.Error(prefix + message);
+            backingLog.Error(Prefix + message);
         }
 
         public void Log(string message)
         {
-            backingLog.Log(prefix + message);
+            backingLog.Log(Prefix + message);
         }
 
         public void AddStringReplace(string from, string to)

Framework/Logging/NullLog.cs

@@ -16,6 +16,7 @@
 
         public override void Error(string message)
         {
+            Console.WriteLine("Error: " + message);
             base.Error(message);
         }
 

Framework/Logging/Stopwatch.cs

@@ -16,19 +16,19 @@ namespace Logging
             this.debug = debug;
         }
 
-        public static void Measure(ILog log, string name, Action action, bool debug = false)
+        public static TimeSpan Measure(ILog log, string name, Action action, bool debug = false)
         {
             var sw = Begin(log, name, debug);
             action();
-            sw.End();
+            return sw.End();
         }
 
-        public static T Measure<T>(ILog log, string name, Func<T> action, bool debug = false)
+        public static StopwatchResult<T> Measure<T>(ILog log, string name, Func<T> action, bool debug = false)
         {
             var sw = Begin(log, name, debug);
             var result = action();
-            sw.End();
-            return result;
+            var duration = sw.End();
+            return new StopwatchResult<T>(result, duration);
         }
 
         public static Stopwatch Begin(ILog log)
@@ -68,4 +68,16 @@ namespace Logging
             return duration;
         }
     }
+
+    public class StopwatchResult<T>
+    {
+        public StopwatchResult(T value, TimeSpan duration)
+        {
+            Value = value;
+            Duration = duration;
+        }
+
+        public T Value { get; }
+        public TimeSpan Duration { get; }
+    }
 }

Framework/NethereumWorkflow/BlockUtils/BlockCache.cs

@@ -0,0 +1,39 @@
+namespace NethereumWorkflow.BlockUtils
+{
+    public class BlockCache
+    {
+        public delegate void CacheClearedEvent();
+
+        private const int MaxEntries = 1024 * 1024 * 5;
+        private readonly Dictionary<ulong, BlockTimeEntry> entries = new Dictionary<ulong, BlockTimeEntry>();
+
+        public event CacheClearedEvent? OnCacheCleared;
+
+        public BlockTimeEntry Add(ulong number, DateTime dateTime)
+        {
+            return Add(new BlockTimeEntry(number, dateTime));
+        }
+
+        public BlockTimeEntry Add(BlockTimeEntry entry)
+        {
+            if (!entries.ContainsKey(entry.BlockNumber))
+            {
+                if (entries.Count > MaxEntries)
+                {
+                    entries.Clear();
+                    var e = OnCacheCleared;
+                    if (e != null) e();
+                }
+                entries.Add(entry.BlockNumber, entry);
+            }
+
+            return entries[entry.BlockNumber];
+        }
+
+        public BlockTimeEntry? Get(ulong number)
+        {
+            if (!entries.TryGetValue(number, out BlockTimeEntry? value)) return null;
+            return value;
+        }
+    }
+}

Framework/NethereumWorkflow/BlockUtils/BlockTimeEntry.cs

@@ -0,0 +1,19 @@
+namespace NethereumWorkflow.BlockUtils
+{
+    public class BlockTimeEntry
+    {
+        public BlockTimeEntry(ulong blockNumber, DateTime utc)
+        {
+            BlockNumber = blockNumber;
+            Utc = utc;
+        }
+
+        public ulong BlockNumber { get; }
+        public DateTime Utc { get; }
+
+        public override string ToString()
+        {
+            return $"[{BlockNumber}] @ {Utc.ToString("o")}";
+        }
+    }
+}

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -0,0 +1,95 @@
+using Logging;
+
+namespace NethereumWorkflow.BlockUtils
+{
+    public class BlockTimeFinder
+    {
+        private readonly BlockCache cache;
+        private readonly BlockchainBounds bounds;
+        private readonly IWeb3Blocks web3;
+        private readonly ILog log;
+
+        public BlockTimeFinder(BlockCache cache, IWeb3Blocks web3, ILog log)
+        {
+            this.web3 = web3;
+            this.log = log;
+
+            this.cache = cache;
+            bounds = new BlockchainBounds(cache, web3);
+        }
+
+        public ulong? GetHighestBlockNumberBefore(DateTime moment)
+        {
+            bounds.Initialize();
+            if (moment <= bounds.Genesis.Utc) return null;
+            if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
+
+            return Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector);
+        }
+
+        public ulong? GetLowestBlockNumberAfter(DateTime moment)
+        {
+            bounds.Initialize();
+            if (moment >= bounds.Current.Utc) return null;
+            if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
+
+            return Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector);
+        }
+
+        private ulong Search(BlockTimeEntry lower, BlockTimeEntry upper, DateTime target, Func<DateTime, BlockTimeEntry, bool> isWhatIwant)
+        {
+            var middle = GetMiddle(lower, upper);
+            if (middle.BlockNumber == lower.BlockNumber)
+            {
+                if (isWhatIwant(target, upper)) return upper.BlockNumber;
+            }
+
+            if (isWhatIwant(target, middle))
+            {
+                return middle.BlockNumber;
+            }
+
+            if (middle.Utc > target)
+            {
+                return Search(lower, middle, target, isWhatIwant);
+            }
+            else
+            {
+                return Search(middle, upper, target, isWhatIwant);
+            }
+        }
+
+        private BlockTimeEntry GetMiddle(BlockTimeEntry lower, BlockTimeEntry upper)
+        {
+            ulong range = upper.BlockNumber - lower.BlockNumber;
+            ulong number = lower.BlockNumber + range / 2;
+            return GetBlock(number);
+        }
+
+        private bool HighestBeforeSelector(DateTime target, BlockTimeEntry entry)
+        {
+            var next = GetBlock(entry.BlockNumber + 1);
+            return
+                entry.Utc < target &&
+                next.Utc > target;
+        }
+
+        private bool LowestAfterSelector(DateTime target, BlockTimeEntry entry)
+        {
+            var previous = GetBlock(entry.BlockNumber - 1);
+            return
+                entry.Utc > target &&
+                previous.Utc < target;
+        }
+
+        private BlockTimeEntry GetBlock(ulong number)
+        {
+            if (number < bounds.Genesis.BlockNumber) throw new Exception("Can't fetch block before genesis.");
+            if (number > bounds.Current.BlockNumber) throw new Exception("Can't fetch block after current.");
+
+            var dateTime = web3.GetTimestampForBlock(number);
+            if (dateTime == null) throw new Exception("Failed to get dateTime for block that should exist.");
+            return cache.Add(number, dateTime.Value);
+        }
+    }
+}

Framework/NethereumWorkflow/BlockUtils/BlockchainBounds.cs

@@ -0,0 +1,106 @@
+namespace NethereumWorkflow.BlockUtils
+{
+    public class BlockchainBounds
+    {
+        private readonly BlockCache cache;
+        private readonly IWeb3Blocks web3;
+
+        public BlockTimeEntry Genesis { get; private set; } = null!;
+        public BlockTimeEntry Current { get; private set; } = null!;
+
+        public BlockchainBounds(BlockCache cache, IWeb3Blocks web3)
+        {
+            this.cache = cache;
+            this.web3 = web3;
+
+            cache.OnCacheCleared += Initialize;
+        }
+
+        public void Initialize()
+        {
+            AddCurrentBlock();
+            LookForGenesisBlock();
+
+            if (Current.BlockNumber == Genesis.BlockNumber)
+            {
+                throw new Exception("Unsupported condition: Current block is genesis block.");
+            }
+        }
+
+        private void LookForGenesisBlock()
+        {
+            if (Genesis != null)
+            {
+                cache.Add(Genesis);
+                return;
+            }
+
+            var blockTime = web3.GetTimestampForBlock(0);
+            if (blockTime != null)
+            {
+                AddGenesisBlock(0, blockTime.Value);
+                return;
+            }
+
+            LookForGenesisBlock(0, Current);
+        }
+
+        private void LookForGenesisBlock(ulong lower, BlockTimeEntry upper)
+        {
+            if (Genesis != null) return;
+
+            var range = upper.BlockNumber - lower;
+            if (range == 1)
+            {
+                var lowTime = web3.GetTimestampForBlock(lower);
+                if (lowTime != null)
+                {
+                    AddGenesisBlock(lower, lowTime.Value);
+                }
+                else
+                {
+                    AddGenesisBlock(upper);
+                }
+                return;
+            }
+
+            var current = lower + range / 2;
+
+            var blockTime = web3.GetTimestampForBlock(current);
+            if (blockTime != null)
+            {
+                var newUpper = cache.Add(current, blockTime.Value);
+                LookForGenesisBlock(lower, newUpper);
+            }
+            else
+            {
+                LookForGenesisBlock(current, upper);
+            }
+        }
+
+        private void AddCurrentBlock()
+        {
+            var currentBlockNumber = web3.GetCurrentBlockNumber();
+            var blockTime = web3.GetTimestampForBlock(currentBlockNumber);
+            if (blockTime == null) throw new Exception("Unable to get dateTime for current block.");
+            AddCurrentBlock(currentBlockNumber, blockTime.Value);
+        }
+
+        private void AddCurrentBlock(ulong currentBlockNumber, DateTime dateTime)
+        {
+            Current = new BlockTimeEntry(currentBlockNumber, dateTime);
+            cache.Add(Current);
+        }
+
+        private void AddGenesisBlock(ulong number, DateTime dateTime)
+        {
+            AddGenesisBlock(new BlockTimeEntry(number, dateTime));
+        }
+
+        private void AddGenesisBlock(BlockTimeEntry entry)
+        {
+            Genesis = entry;
+            cache.Add(Genesis);
+        }
+    }
+}

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -1,13 +1,18 @@
 using Logging;
+using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
+using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace NethereumWorkflow
 {
     public class NethereumInteraction
     {
+        // BlockCache is a static instance: It stays alive for the duration of the application runtime.
+        private readonly static BlockCache blockCache = new BlockCache();
+
         private readonly ILog log;
         private readonly Web3 web3;
 
@@ -17,11 +22,12 @@ namespace NethereumWorkflow
             this.web3 = web3;
         }
 
-        public void SendEth(string toAddress, decimal ethAmount)
+        public string SendEth(string toAddress, decimal ethAmount)
         {
             log.Debug();
             var receipt = Time.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(toAddress, ethAmount));
             if (!receipt.Succeeded()) throw new Exception("Unable to send Eth");
+            return receipt.TransactionHash;
         }
 
         public decimal GetEthBalance()
@@ -44,12 +50,19 @@ namespace NethereumWorkflow
             return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function));
         }
 
-        public void SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
+        public string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
         {
             log.Debug();
             var handler = web3.Eth.GetContractTransactionHandler<TFunction>();
             var receipt = Time.Wait(handler.SendRequestAndWaitForReceiptAsync(contractAddress, function));
             if (!receipt.Succeeded()) throw new Exception("Unable to perform contract transaction.");
+            return receipt.TransactionHash;
+        }
+
+        public Transaction GetTransaction(string transactionHash)
+        {
+            log.Debug();
+            return Time.Wait(web3.Eth.Transactions.GetTransactionByHash.SendRequestAsync(transactionHash));
         }
 
         public decimal? GetSyncedBlockNumber()
@@ -75,5 +88,36 @@ namespace NethereumWorkflow
                 return false;
             }
         }
+
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new()
+        {
+            var wrapper = new Web3Wrapper(web3, log);
+            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
+
+            var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From);
+            var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To);
+
+            if (!fromBlock.HasValue)
+            {
+                log.Error("Failed to find lowest block for time range: " + timeRange);
+                throw new Exception("Failed");
+            }
+            if (!toBlock.HasValue)
+            {
+                log.Error("Failed to find highest block for time range: " + timeRange);
+                throw new Exception("Failed");
+            }
+
+            return GetEvents<TEvent>(address, fromBlock.Value, toBlock.Value);
+        }
+
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new()
+        {
+            var eventHandler = web3.Eth.GetEvent<TEvent>(address);
+            var from = new BlockParameter(fromBlockNumber);
+            var to = new BlockParameter(toBlockNumber);
+            var blockFilter = Time.Wait(eventHandler.CreateFilterBlockRangeAsync(from, to));
+            return Time.Wait(eventHandler.GetAllChangesAsync(blockFilter));
+        }
     }
 }

Framework/NethereumWorkflow/Web3Wrapper.cs

@@ -0,0 +1,46 @@
+using Logging;
+using Nethereum.RPC.Eth.DTOs;
+using Nethereum.Web3;
+using Utils;
+
+namespace NethereumWorkflow
+{
+    public interface IWeb3Blocks
+    {
+        ulong GetCurrentBlockNumber();
+        DateTime? GetTimestampForBlock(ulong blockNumber);
+    }
+
+    public class Web3Wrapper : IWeb3Blocks
+    {
+        private readonly Web3 web3;
+        private readonly ILog log;
+
+        public Web3Wrapper(Web3 web3, ILog log)
+        {
+            this.web3 = web3;
+            this.log = log;
+        }
+
+        public ulong GetCurrentBlockNumber()
+        {
+            var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
+            return Convert.ToUInt64(number.ToDecimal());
+        }
+
+        public DateTime? GetTimestampForBlock(ulong blockNumber)
+        {
+            try
+            {
+                var block = Time.Wait(web3.Eth.Blocks.GetBlockWithTransactionsByNumber.SendRequestAsync(new BlockParameter(blockNumber)));
+                if (block == null) return null;
+                return DateTimeOffset.FromUnixTimeSeconds(Convert.ToInt64(block.Timestamp.ToDecimal())).UtcDateTime;
+            }
+            catch (Exception ex)
+            {
+                log.Error("Exception while getting timestamp for block: " + ex);
+                return null;
+            }
+        }
+    }
+}

Framework/Utils/ByteSize.cs

@@ -2,6 +2,9 @@
 {
     public class ByteSize
     {
+        public static readonly ByteSize Zero = new ByteSize(0);
+        public const double DefaultSecondsPerMB = 10.0;
+
         public ByteSize(long sizeInBytes)
         {
             if (sizeInBytes < 0) throw new ArgumentException("Cannot create ByteSize object with size less than 0. Was: " + sizeInBytes);
@@ -10,7 +13,6 @@
 
         public long SizeInBytes { get; }
 
-        public const double DefaultSecondsPerMB = 10.0;
 
         public long ToMB()
         {
@@ -46,6 +48,18 @@
         }
     }
 
+    public class BytesPerSecond : ByteSize
+    {
+        public BytesPerSecond(long sizeInBytes) : base(sizeInBytes)
+        {
+        }
+
+        public override string ToString()
+        {
+            return base.ToString() + "/s";
+        }
+    }
+
     public static class ByteSizeIntExtensions
     {
         private const long Kilo = 1024;

Framework/Utils/TaskFactory.cs

@@ -1,15 +1,27 @@
-namespace ContinuousTests
+namespace Utils
 {
     public class TaskFactory
     {
         private readonly object taskLock = new();
         private readonly List<Task> activeTasks = new List<Task>();
 
-        public void Run(Action action)
+        public void Run(Action action, string name)
         {
             lock (taskLock)
             {
-                activeTasks.Add(Task.Run(action).ContinueWith(CleanupTask, null));
+                activeTasks.Add(Task.Run(() => CatchException(action, name)).ContinueWith(CleanupTask, null));
+            }
+        }
+
+        private void CatchException(Action action, string name)
+        {
+            try
+            {
+                action();
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Exception in task '{name}': " + ex);
             }
         }
 

Framework/Utils/Time.cs

@@ -13,6 +13,11 @@
             return task.Result;
         }
 
+        public static void Wait(Task task)
+        {
+            task.Wait();
+        }
+
         public static string FormatDuration(TimeSpan d)
         {
             var result = "";
@@ -23,6 +28,35 @@
             return result;
         }
         
+        public static TimeSpan ParseTimespan(string span)
+        {
+            span = span.Replace(" ", "").Replace(",", "");
+            var result = TimeSpan.Zero;
+            var number = "";
+            foreach (var c in span)
+            {
+                if (char.IsNumber(c)) number += c;
+                else
+                {
+                    var value = Convert.ToInt32(number);
+                    number = "";
+
+                    if (c == 'd') result += TimeSpan.FromDays(value);
+                    else if (c == 'h') result += TimeSpan.FromHours(value);
+                    else if (c == 'm') result += TimeSpan.FromMinutes(value);
+                    else if (c == 's') result += TimeSpan.FromSeconds(value);
+                    else throw new Exception("Unknown time modifier: " + c);
+                }
+            }
+
+            if (!string.IsNullOrEmpty(number))
+            {
+                var value = Convert.ToInt32(number);
+                result += TimeSpan.FromSeconds(value);
+            }
+            return result;
+        }
+
         public static void WaitUntil(Func<bool> predicate)
         {
             WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));

Framework/Utils/TimeRange.cs

@@ -0,0 +1,24 @@
+namespace Utils
+{
+    public class TimeRange
+    {
+        public TimeRange(DateTime from, DateTime to)
+        {
+            if (from < to)
+            {
+                From = from;
+                To = to;
+            }
+            else
+            {
+                From = to;
+                To = from;
+            }
+            Duration = To - From;
+        }
+
+        public DateTime From { get; }
+        public DateTime To { get; }
+        public TimeSpan Duration { get; }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -1,5 +1,11 @@
-using GethPlugin;
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
 using Logging;
+using Nethereum.ABI;
+using Nethereum.Hex.HexTypes;
+using Nethereum.Util;
+using NethereumWorkflow;
+using Utils;
 
 namespace CodexContractsPlugin
 {
@@ -7,10 +13,28 @@ namespace CodexContractsPlugin
     {
         CodexContractsDeployment Deployment { get; }
 
-        void MintTestTokens(IHasEthAddress owner, TestToken testTokens);
-        void MintTestTokens(EthAddress ethAddress, TestToken testTokens);
+        bool IsDeployed();
+        string MintTestTokens(IHasEthAddress owner, TestToken testTokens);
+        string MintTestTokens(EthAddress ethAddress, TestToken testTokens);
         TestToken GetTestTokenBalance(IHasEthAddress owner);
         TestToken GetTestTokenBalance(EthAddress ethAddress);
+
+        Request[] GetStorageRequests(TimeRange timeRange);
+        EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
+        RequestState GetRequestState(Request request);
+        RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange);
+        RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange);
+        SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange);
+        SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange);
+    }
+
+    public enum RequestState
+    {
+        New,
+        Started,
+        Cancelled,
+        Finished,
+        Failed
     }
 
     public class CodexContractsAccess : ICodexContracts
@@ -27,15 +51,19 @@ namespace CodexContractsPlugin
 
         public CodexContractsDeployment Deployment { get; }
 
-        public void MintTestTokens(IHasEthAddress owner, TestToken testTokens)
+        public bool IsDeployed()
         {
-            MintTestTokens(owner.EthAddress, testTokens);
+            return !string.IsNullOrEmpty(StartInteraction().GetTokenName(Deployment.TokenAddress));
         }
 
-        public void MintTestTokens(EthAddress ethAddress, TestToken testTokens)
+        public string MintTestTokens(IHasEthAddress owner, TestToken testTokens)
         {
-            var interaction = new ContractInteractions(log, gethNode);
-            interaction.MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
+            return MintTestTokens(owner.EthAddress, testTokens);
+        }
+
+        public string MintTestTokens(EthAddress ethAddress, TestToken testTokens)
+        {
+            return StartInteraction().MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
         }
 
         public TestToken GetTestTokenBalance(IHasEthAddress owner)
@@ -45,9 +73,108 @@ namespace CodexContractsPlugin
 
         public TestToken GetTestTokenBalance(EthAddress ethAddress)
         {
-            var interaction = new ContractInteractions(log, gethNode);
-            var balance = interaction.GetBalance(Deployment.TokenAddress, ethAddress.Address);
+            var balance = StartInteraction().GetBalance(Deployment.TokenAddress, ethAddress.Address);
             return balance.TestTokens();
         }
+
+        public Request[] GetStorageRequests(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<StorageRequestedEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            var i = StartInteraction();
+            return events
+                    .Select(e =>
+                    {
+                        var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId);
+                        var result = requestEvent.ReturnValue1;
+                        result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                        result.RequestId = e.Event.RequestId;
+                        return result;
+                    })
+                    .ToArray();
+        }
+
+        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<RequestCancelledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<SlotFilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<SlotFreedEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
+        {
+            var encoder = new ABIEncode();
+            var encoded = encoder.GetABIEncoded(
+                new ABIValue("bytes32", storageRequest.RequestId),
+                new ABIValue("uint256", slotIndex.ToBig())
+            );
+
+            var hashed = Sha3Keccack.Current.CalculateHash(encoded);
+
+            var func = new GetHostFunction
+            {
+                SlotId = hashed
+            };
+            var address = gethNode.Call<GetHostFunction, string>(Deployment.MarketplaceAddress, func);
+            if (string.IsNullOrEmpty(address)) return null;
+            return new EthAddress(address);
+        }
+
+        public RequestState GetRequestState(Request request)
+        {
+            var func = new RequestStateFunction
+            {
+                RequestId = request.RequestId
+            };
+            return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
+        }
+
+        private EthAddress GetEthAddressFromTransaction(string transactionHash)
+        {
+            var transaction = gethNode.GetTransaction(transactionHash);
+            return new EthAddress(transaction.From);
+        }
+
+        private ContractInteractions StartInteraction()
+        {
+            return new ContractInteractions(log, gethNode);
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexContractsPlugin
 {
     public class CodexContractsContainerRecipe : ContainerRecipeFactory
     {
-        public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests";
+        public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:latest-dist-tests";
 
         public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
         public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
@@ -21,6 +21,8 @@ namespace CodexContractsPlugin
 
             var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
 
+            SetSchedulingAffinity(notIn: "false");
+
             AddEnvVar("DISTTEST_NETWORK_URL", address.ToString());
             AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork");
             AddEnvVar("KEEP_ALIVE", "1");

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -33,7 +33,7 @@ namespace CodexContractsPlugin
             try
             {
                 var result = DeployContract(container, workflow, gethNode);
-                workflow.Stop(containers);
+                workflow.Stop(containers, waitTillStopped: false);
                 Log("Container stopped.");
                 return result;
             }
@@ -85,7 +85,7 @@ namespace CodexContractsPlugin
 
         private void WaitUntil(Func<bool> predicate)
         {
-            Time.WaitUntil(predicate, TimeSpan.FromMinutes(3), TimeSpan.FromSeconds(2));
+            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2));
         }
 
         private StartupConfig CreateStartupConfig(IGethNode gethNode)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -1,7 +1,9 @@
-using GethPlugin;
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
 using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
+using Nethereum.Hex.HexConvertors.Extensions;
 using NethereumWorkflow;
 using System.Numerics;
 
@@ -26,10 +28,26 @@ namespace CodexContractsPlugin
             return gethNode.Call<GetTokenFunction, string>(marketplaceAddress, function);
         }
 
-        public void MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
+        public string GetTokenName(string tokenAddress)
+        {
+            try
+            {
+                log.Debug(tokenAddress);
+                var function = new GetTokenNameFunction();
+
+                return gethNode.Call<GetTokenNameFunction, string>(tokenAddress, function);
+            }
+            catch (Exception ex)
+            {
+                log.Log("Failed to get token name: " + ex);
+                return string.Empty;
+            }
+        }
+
+        public string MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
         {
             log.Debug($"{amount} -> {address} (token: {tokenAddress})");
-            MintTokens(address.Address, amount, tokenAddress);
+            return MintTokens(address.Address, amount, tokenAddress);
         }
 
         public decimal GetBalance(string tokenAddress, string account)
@@ -43,6 +61,17 @@ namespace CodexContractsPlugin
             return gethNode.Call<GetTokenBalanceFunction, BigInteger>(tokenAddress, function).ToDecimal();
         }
 
+        public GetRequestOutputDTO GetRequest(string marketplaceAddress, byte[] requestId)
+        {
+
+            log.Debug($"({marketplaceAddress}) {requestId.ToHex(true)}");
+            var func = new GetRequestFunction
+            {
+                RequestId = requestId
+            };
+            return gethNode.Call<GetRequestFunction, GetRequestOutputDTO>(marketplaceAddress, func);
+        }
+
         public bool IsSynced(string marketplaceAddress, string marketplaceAbi)
         {
             log.Debug();
@@ -56,7 +85,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        private void MintTokens(string account, decimal amount, string tokenAddress)
+        private string MintTokens(string account, decimal amount, string tokenAddress)
         {
             log.Debug($"({tokenAddress}) {amount} --> {account}");
             if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
@@ -67,7 +96,7 @@ namespace CodexContractsPlugin
                 Amount = amount.ToBig()
             };
 
-            gethNode.SendTransaction(tokenAddress, function);
+            return gethNode.SendTransaction(tokenAddress, function);
         }
 
         private bool IsBlockNumberOK()
@@ -87,6 +116,11 @@ namespace CodexContractsPlugin
     {
     }
 
+    [Function("name", "string")]
+    public class GetTokenNameFunction : FunctionMessage
+    {
+    }
+
     [Function("mint")]
     public class MintTokensFunction : FunctionMessage
     {

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -0,0 +1,35 @@
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
+using GethPlugin;
+
+namespace CodexContractsPlugin.Marketplace
+{
+    public partial class Request : RequestBase
+    {
+        public ulong BlockNumber { get; set; }
+        public byte[] RequestId { get; set; }
+
+        public EthAddress ClientAddress { get { return new EthAddress(Client); } }
+    }
+
+    public partial class RequestFulfilledEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+    }
+
+    public partial class RequestCancelledEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+    }
+
+    public partial class SlotFilledEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+        public EthAddress Host { get; set; }
+    }
+
+    public partial class SlotFreedEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+    }
+}
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexContractsPlugin/Marketplace/Marketplace.cs

@@ -0,0 +1,517 @@
+using Nethereum.ABI.FunctionEncoding.Attributes;
+using Nethereum.Contracts;
+using System.Numerics;
+
+// Generated code, do not modify.
+
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
+namespace CodexContractsPlugin.Marketplace
+{
+    public partial class ConfigFunction : ConfigFunctionBase { }
+
+    [Function("config", typeof(ConfigOutputDTO))]
+    public class ConfigFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class FillSlotFunction : FillSlotFunctionBase { }
+
+    [Function("fillSlot")]
+    public class FillSlotFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("uint256", "slotIndex", 2)]
+        public virtual BigInteger SlotIndex { get; set; }
+        [Parameter("bytes", "proof", 3)]
+        public virtual byte[] Proof { get; set; }
+    }
+
+    public partial class FreeSlotFunction : FreeSlotFunctionBase { }
+
+    [Function("freeSlot")]
+    public class FreeSlotFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class GetActiveSlotFunction : GetActiveSlotFunctionBase { }
+
+    [Function("getActiveSlot", typeof(GetActiveSlotOutputDTO))]
+    public class GetActiveSlotFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class GetChallengeFunction : GetChallengeFunctionBase { }
+
+    [Function("getChallenge", "bytes32")]
+    public class GetChallengeFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class GetHostFunction : GetHostFunctionBase { }
+
+    [Function("getHost", "address")]
+    public class GetHostFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class GetPointerFunction : GetPointerFunctionBase { }
+
+    [Function("getPointer", "uint8")]
+    public class GetPointerFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class GetRequestFunction : GetRequestFunctionBase { }
+
+    [Function("getRequest", typeof(GetRequestOutputDTO))]
+    public class GetRequestFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class IsProofRequiredFunction : IsProofRequiredFunctionBase { }
+
+    [Function("isProofRequired", "bool")]
+    public class IsProofRequiredFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class MarkProofAsMissingFunction : MarkProofAsMissingFunctionBase { }
+
+    [Function("markProofAsMissing")]
+    public class MarkProofAsMissingFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+        [Parameter("uint256", "period", 2)]
+        public virtual BigInteger Period { get; set; }
+    }
+
+    public partial class MissingProofsFunction : MissingProofsFunctionBase { }
+
+    [Function("missingProofs", "uint256")]
+    public class MissingProofsFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class MyRequestsFunction : MyRequestsFunctionBase { }
+
+    [Function("myRequests", "bytes32[]")]
+    public class MyRequestsFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class MySlotsFunction : MySlotsFunctionBase { }
+
+    [Function("mySlots", "bytes32[]")]
+    public class MySlotsFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class RequestEndFunction : RequestEndFunctionBase { }
+
+    [Function("requestEnd", "uint256")]
+    public class RequestEndFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestStateFunction : RequestStateFunctionBase { }
+
+    [Function("requestState", "uint8")]
+    public class RequestStateFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestStorageFunction : RequestStorageFunctionBase { }
+
+    [Function("requestStorage")]
+    public class RequestStorageFunctionBase : FunctionMessage
+    {
+        [Parameter("tuple", "request", 1)]
+        public virtual Request Request { get; set; }
+    }
+
+    public partial class SlotStateFunction : SlotStateFunctionBase { }
+
+    [Function("slotState", "uint8")]
+    public class SlotStateFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class SubmitProofFunction : SubmitProofFunctionBase { }
+
+    [Function("submitProof")]
+    public class SubmitProofFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+        [Parameter("bytes", "proof", 2)]
+        public virtual byte[] Proof { get; set; }
+    }
+
+    public partial class TokenFunction : TokenFunctionBase { }
+
+    [Function("token", "address")]
+    public class TokenFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class WillProofBeRequiredFunction : WillProofBeRequiredFunctionBase { }
+
+    [Function("willProofBeRequired", "bool")]
+    public class WillProofBeRequiredFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class WithdrawFundsFunction : WithdrawFundsFunctionBase { }
+
+    [Function("withdrawFunds")]
+    public class WithdrawFundsFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class ProofSubmittedEventDTO : ProofSubmittedEventDTOBase { }
+
+    [Event("ProofSubmitted")]
+    public class ProofSubmittedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "id", 1, false)]
+        public virtual byte[] Id { get; set; }
+        [Parameter("bytes", "proof", 2, false)]
+        public virtual byte[] Proof { get; set; }
+    }
+
+    public partial class RequestCancelledEventDTO : RequestCancelledEventDTOBase { }
+
+    [Event("RequestCancelled")]
+    public class RequestCancelledEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestFailedEventDTO : RequestFailedEventDTOBase { }
+
+    [Event("RequestFailed")]
+    public class RequestFailedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestFulfilledEventDTO : RequestFulfilledEventDTOBase { }
+
+    [Event("RequestFulfilled")]
+    public class RequestFulfilledEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class SlotFilledEventDTO : SlotFilledEventDTOBase { }
+
+    [Event("SlotFilled")]
+    public class SlotFilledEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("uint256", "slotIndex", 2, false)]
+        public virtual BigInteger SlotIndex { get; set; }
+    }
+
+    public partial class SlotFreedEventDTO : SlotFreedEventDTOBase { }
+
+    [Event("SlotFreed")]
+    public class SlotFreedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("uint256", "slotIndex", 2, false)]
+        public virtual BigInteger SlotIndex { get; set; }
+    }
+
+    public partial class StorageRequestedEventDTO : StorageRequestedEventDTOBase { }
+
+    [Event("StorageRequested")]
+    public class StorageRequestedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, false)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("tuple", "ask", 2, false)]
+        public virtual Ask Ask { get; set; }
+        [Parameter("uint256", "expiry", 3, false)]
+        public virtual BigInteger Expiry { get; set; }
+    }
+
+    public partial class ConfigOutputDTO : ConfigOutputDTOBase { }
+
+    [FunctionOutput]
+    public class ConfigOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("tuple", "collateral", 1)]
+        public virtual CollateralConfig Collateral { get; set; }
+        [Parameter("tuple", "proofs", 2)]
+        public virtual ProofConfig Proofs { get; set; }
+    }
+
+
+
+
+
+    public partial class GetActiveSlotOutputDTO : GetActiveSlotOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetActiveSlotOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("tuple", "", 1)]
+        public virtual ActiveSlot ReturnValue1 { get; set; }
+    }
+
+    public partial class GetChallengeOutputDTO : GetChallengeOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetChallengeOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bytes32", "", 1)]
+        public virtual byte[] ReturnValue1 { get; set; }
+    }
+
+    public partial class GetHostOutputDTO : GetHostOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetHostOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("address", "", 1)]
+        public virtual string ReturnValue1 { get; set; }
+    }
+
+    public partial class GetPointerOutputDTO : GetPointerOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetPointerOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint8", "", 1)]
+        public virtual byte ReturnValue1 { get; set; }
+    }
+
+    public partial class GetRequestOutputDTO : GetRequestOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetRequestOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("tuple", "", 1)]
+        public virtual Request ReturnValue1 { get; set; }
+    }
+
+    public partial class IsProofRequiredOutputDTO : IsProofRequiredOutputDTOBase { }
+
+    [FunctionOutput]
+    public class IsProofRequiredOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bool", "", 1)]
+        public virtual bool ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class MissingProofsOutputDTO : MissingProofsOutputDTOBase { }
+
+    [FunctionOutput]
+    public class MissingProofsOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint256", "", 1)]
+        public virtual BigInteger ReturnValue1 { get; set; }
+    }
+
+    public partial class MyRequestsOutputDTO : MyRequestsOutputDTOBase { }
+
+    [FunctionOutput]
+    public class MyRequestsOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bytes32[]", "", 1)]
+        public virtual List<byte[]> ReturnValue1 { get; set; }
+    }
+
+    public partial class MySlotsOutputDTO : MySlotsOutputDTOBase { }
+
+    [FunctionOutput]
+    public class MySlotsOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bytes32[]", "", 1)]
+        public virtual List<byte[]> ReturnValue1 { get; set; }
+    }
+
+    public partial class RequestEndOutputDTO : RequestEndOutputDTOBase { }
+
+    [FunctionOutput]
+    public class RequestEndOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint256", "", 1)]
+        public virtual BigInteger ReturnValue1 { get; set; }
+    }
+
+    public partial class RequestStateOutputDTO : RequestStateOutputDTOBase { }
+
+    [FunctionOutput]
+    public class RequestStateOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint8", "", 1)]
+        public virtual byte ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class SlotStateOutputDTO : SlotStateOutputDTOBase { }
+
+    [FunctionOutput]
+    public class SlotStateOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint8", "", 1)]
+        public virtual byte ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class TokenOutputDTO : TokenOutputDTOBase { }
+
+    [FunctionOutput]
+    public class TokenOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("address", "", 1)]
+        public virtual string ReturnValue1 { get; set; }
+    }
+
+    public partial class WillProofBeRequiredOutputDTO : WillProofBeRequiredOutputDTOBase { }
+
+    [FunctionOutput]
+    public class WillProofBeRequiredOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bool", "", 1)]
+        public virtual bool ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class CollateralConfig : CollateralConfigBase { }
+
+    public class CollateralConfigBase
+    {
+        [Parameter("uint8", "repairRewardPercentage", 1)]
+        public virtual byte RepairRewardPercentage { get; set; }
+        [Parameter("uint8", "maxNumberOfSlashes", 2)]
+        public virtual byte MaxNumberOfSlashes { get; set; }
+        [Parameter("uint16", "slashCriterion", 3)]
+        public virtual ushort SlashCriterion { get; set; }
+        [Parameter("uint8", "slashPercentage", 4)]
+        public virtual byte SlashPercentage { get; set; }
+    }
+
+    public partial class ProofConfig : ProofConfigBase { }
+
+    public class ProofConfigBase
+    {
+        [Parameter("uint256", "period", 1)]
+        public virtual BigInteger Period { get; set; }
+        [Parameter("uint256", "timeout", 2)]
+        public virtual BigInteger Timeout { get; set; }
+        [Parameter("uint8", "downtime", 3)]
+        public virtual byte Downtime { get; set; }
+    }
+
+    public partial class MarketplaceConfig : MarketplaceConfigBase { }
+
+    public class MarketplaceConfigBase
+    {
+        [Parameter("tuple", "collateral", 1)]
+        public virtual CollateralConfig Collateral { get; set; }
+        [Parameter("tuple", "proofs", 2)]
+        public virtual ProofConfig Proofs { get; set; }
+    }
+
+    public partial class Ask : AskBase { }
+
+    public class AskBase
+    {
+        [Parameter("uint64", "slots", 1)]
+        public virtual ulong Slots { get; set; }
+        [Parameter("uint256", "slotSize", 2)]
+        public virtual BigInteger SlotSize { get; set; }
+        [Parameter("uint256", "duration", 3)]
+        public virtual BigInteger Duration { get; set; }
+        [Parameter("uint256", "proofProbability", 4)]
+        public virtual BigInteger ProofProbability { get; set; }
+        [Parameter("uint256", "reward", 5)]
+        public virtual BigInteger Reward { get; set; }
+        [Parameter("uint256", "collateral", 6)]
+        public virtual BigInteger Collateral { get; set; }
+        [Parameter("uint64", "maxSlotLoss", 7)]
+        public virtual ulong MaxSlotLoss { get; set; }
+    }
+
+    public partial class Content : ContentBase { }
+
+    public class ContentBase
+    {
+        [Parameter("string", "cid", 1)]
+        public virtual string Cid { get; set; }
+        [Parameter("bytes32", "merkleRoot", 2)]
+        public virtual byte[] MerkleRoot { get; set; }
+    }
+
+    public partial class Request : RequestBase { }
+
+    public class RequestBase
+    {
+        [Parameter("address", "client", 1)]
+        public virtual string Client { get; set; }
+        [Parameter("tuple", "ask", 2)]
+        public virtual Ask Ask { get; set; }
+        [Parameter("tuple", "content", 3)]
+        public virtual Content Content { get; set; }
+        [Parameter("uint256", "expiry", 4)]
+        public virtual BigInteger Expiry { get; set; }
+        [Parameter("bytes32", "nonce", 5)]
+        public virtual byte[] Nonce { get; set; }
+    }
+
+    public partial class ActiveSlot : ActiveSlotBase { }
+
+    public class ActiveSlotBase
+    {
+        [Parameter("tuple", "request", 1)]
+        public virtual Request Request { get; set; }
+        [Parameter("uint256", "slotIndex", 2)]
+        public virtual BigInteger SlotIndex { get; set; }
+    }
+}
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexContractsPlugin/Marketplace/README.md

@@ -0,0 +1 @@
+This code was generated using the Nethereum code generator, here: http://playground.nethereum.com

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs

@@ -13,16 +13,30 @@ namespace CodexDiscordBotPlugin
         {
             var config = startupConfig.Get<DiscordBotStartupConfig>();
 
+            SetSchedulingAffinity(notIn: "false");
+
             AddEnvVar("TOKEN", config.Token);
             AddEnvVar("SERVERNAME", config.ServerName);
             AddEnvVar("ADMINROLE", config.AdminRoleName);
             AddEnvVar("ADMINCHANNELNAME", config.AdminChannelName);
+            AddEnvVar("KUBECONFIG", "/opt/kubeconfig.yaml");
+            AddEnvVar("KUBENAMESPACE", config.KubeNamespace);
+
+            var gethInfo = config.GethInfo;
+            AddEnvVar("GETH_HOST", gethInfo.Host);
+            AddEnvVar("GETH_HTTP_PORT", gethInfo.Port.ToString());
+            AddEnvVar("GETH_PRIVATE_KEY", gethInfo.PrivKey);
+            AddEnvVar("CODEXCONTRACTS_MARKETPLACEADDRESS", gethInfo.MarketplaceAddress);
+            AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress);
+            AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi);
 
             if (!string.IsNullOrEmpty(config.DataPath))
             {
                 AddEnvVar("DATAPATH", config.DataPath);
                 AddVolume(config.DataPath, 1.GB());
             }
+
+            AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml", secret: "discordbot-sa-kubeconfig");
         }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs

@@ -2,13 +2,15 @@
 {
     public class DiscordBotStartupConfig
     {
-        public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName)
+        public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName, string kubeNamespace, DiscordBotGethInfo gethInfo)
         {
             Name = name;
             Token = token;
             ServerName = serverName;
             AdminRoleName = adminRoleName;
             AdminChannelName = adminChannelName;
+            KubeNamespace = kubeNamespace;
+            GethInfo = gethInfo;
         }
 
         public string Name { get; }
@@ -16,6 +18,28 @@
         public string ServerName { get; }
         public string AdminRoleName { get; }
         public string AdminChannelName { get; }
+        public string KubeNamespace { get; }
+        public DiscordBotGethInfo GethInfo { get; }
         public string? DataPath { get; set; }
     }
+
+    public class DiscordBotGethInfo
+    {
+        public DiscordBotGethInfo(string host, int port, string privKey, string marketplaceAddress, string tokenAddress, string abi)
+        {
+            Host = host;
+            Port = port;
+            PrivKey = privKey;
+            MarketplaceAddress = marketplaceAddress;
+            TokenAddress = tokenAddress;
+            Abi = abi;
+        }
+
+        public string Host { get; }
+        public int Port { get; }
+        public string PrivKey { get; }
+        public string MarketplaceAddress { get; }
+        public string TokenAddress { get; }
+        public string Abi { get; }
+    }
 }

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -0,0 +1,92 @@
+using Core;
+using KubernetesWorkflow.Types;
+using Logging;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace CodexPlugin
+{
+    public class ApiChecker
+    {
+        // <INSERT-OPENAPI-YAML-HASH>
+        private const string OpenApiYamlHash = "DC-90-1B-63-76-1B-92-01-05-65-33-DA-17-C2-34-83-E1-2E-6C-A9-04-4D-68-ED-96-43-F5-E5-6A-00-0F-5F";
+        private const string OpenApiFilePath = "/codex/openapi.yaml";
+        private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
+
+        private const bool Disable = false;
+
+        private const string Warning =
+            "Warning: CodexPlugin was unable to find the openapi.yaml file in the Codex container. Are you running an old version of Codex? " +
+            "Plugin will continue as normal, but API compatibility is not guaranteed!";
+
+        private const string Failure =
+            "Codex API compatibility check failed! " +
+            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. Please update the openapi.yaml in " +
+            "'ProjectPlugins/CodexPlugin' and rebuild this project. If you wish to disable API compatibility checking, please set " +
+            $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/CodexPlugin/ApiChecker.cs'.";
+
+        private static bool checkPassed = false;
+
+        private readonly IPluginTools pluginTools;
+        private readonly ILog log;
+
+        public ApiChecker(IPluginTools pluginTools)
+        {
+            this.pluginTools = pluginTools;
+            log = pluginTools.GetLog();
+
+            if (string.IsNullOrEmpty(OpenApiYamlHash)) throw new Exception("OpenAPI yaml hash was not inserted by pre-build trigger.");
+        }
+
+        public void CheckCompatibility(RunningContainers[] containers)
+        {
+            if (checkPassed) return;
+
+            Log("CodexPlugin is checking API compatibility...");
+
+            if (Disable || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(DisableEnvironmentVariable)))
+            {
+                Log("API compatibility checking has been disabled.");
+                checkPassed = true;
+                return;
+            }
+
+            var workflow = pluginTools.CreateWorkflow();
+            var container = containers.First().Containers.First();
+            var containerApi = workflow.ExecuteCommand(container, "cat", OpenApiFilePath);
+
+            if (string.IsNullOrEmpty(containerApi))
+            {
+                log.Error(Warning);
+
+                checkPassed = true;
+                return;
+            }
+
+            var containerHash = Hash(containerApi);
+            if (containerHash == OpenApiYamlHash)
+            {
+                Log("API compatibility check passed.");
+                checkPassed = true;
+                return;
+            }
+
+            log.Error(Failure);
+            throw new Exception(Failure);
+        }
+
+        private string Hash(string file)
+        {
+            var fileBytes = Encoding.ASCII.GetBytes(file
+                .Replace(Environment.NewLine, ""));
+            var sha = SHA256.Create();
+            var hash = sha.ComputeHash(fileBytes);
+            return BitConverter.ToString(hash);
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -1,4 +1,5 @@
-using Core;
+using CodexOpenApi;
+using Core;
 using KubernetesWorkflow;
 using KubernetesWorkflow.Types;
 using Utils;
@@ -8,6 +9,7 @@ namespace CodexPlugin
     public class CodexAccess : ILogHandler
     {
         private readonly IPluginTools tools;
+        private readonly Mapper mapper = new Mapper();
         private bool hasContainerCrashed;
 
         public CodexAccess(IPluginTools tools, RunningContainer container, CrashWatcher crashWatcher)
@@ -23,77 +25,72 @@ namespace CodexPlugin
         public RunningContainer Container { get; }
         public CrashWatcher CrashWatcher { get; }
 
-        public CodexDebugResponse GetDebugInfo()
+        public DebugInfo GetDebugInfo()
         {
-            return Http().HttpGetJson<CodexDebugResponse>("debug/info");
+            return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
         }
 
-        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        public DebugPeer GetDebugPeer(string peerId)
         {
-            var http = Http();
-            var str = http.HttpGetString($"debug/peer/{peerId}");
+            // Cannot use openAPI: debug/peer endpoint is not specified there.
+            var endpoint = GetEndpoint();
+            var str = endpoint.HttpGetString($"debug/peer/{peerId}");
 
             if (str.ToLowerInvariant() == "unable to find peer!")
             {
-                return new CodexDebugPeerResponse
+                return new DebugPeer
                 {
                     IsPeerFound = false
                 };
             }
 
-            var result = http.Deserialize<CodexDebugPeerResponse>(str);
+            var result = endpoint.Deserialize<DebugPeer>(str);
             result.IsPeerFound = true;
             return result;
         }
 
-        public CodexDebugBlockExchangeResponse GetDebugBlockExchange()
+        public void ConnectToPeer(string peerId, string[] peerMultiAddresses)
         {
-            return Http().HttpGetJson<CodexDebugBlockExchangeResponse>("debug/blockexchange");
-        }
-
-        public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
-        {
-            return LongHttp().HttpGetJson<CodexDebugRepoStoreResponse[]>("debug/repostore");
-        }
-
-        public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
-        {
-            return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");
+            OnCodex(api =>
+            {
+                Time.Wait(api.ConnectPeerAsync(peerId, peerMultiAddresses));
+                return Task.FromResult(string.Empty);
+            });
         }
 
         public string UploadFile(FileStream fileStream)
         {
-            return Http().HttpPostStream("data", fileStream);
+            return OnCodex(api => api.UploadAsync(fileStream));
         }
 
         public Stream DownloadFile(string contentId)
         {
-            return Http().HttpGetStream("data/" + contentId);
+            var fileResponse = OnCodex(api => api.DownloadNetworkAsync(contentId));
+            if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
+            return fileResponse.Stream;
         }
 
-        public CodexLocalDataResponse[] LocalFiles()
+        public LocalDatasetList LocalFiles()
         {
-            return Http().HttpGetJson<CodexLocalDataResponse[]>("local");
+            return mapper.Map(OnCodex(api => api.ListDataAsync()));
         }
 
-        public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
+        public StorageAvailability SalesAvailability(StorageAvailability request)
         {
-            return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
+            var body = mapper.Map(request);
+            var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
+            return mapper.Map(read);
         }
 
-        public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
+        public string RequestStorage(StoragePurchaseRequest request)
         {
-            return Http().HttpPostJson($"storage/request/{contentId}", request);
+            var body = mapper.Map(request);
+            return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
-        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
+        public StoragePurchase GetPurchaseStatus(string purchaseId)
         {
-            return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
-        }
-
-        public string ConnectToPeer(string peerId, string peerMultiAddress)
-        {
-            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
+            return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
         }
 
         public string GetName()
@@ -107,14 +104,24 @@ namespace CodexPlugin
             return workflow.GetPodInfo(Container);
         }
 
-        private IHttp Http()
+        private T OnCodex<T>(Func<CodexApi, Task<T>> action)
         {
-            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
+            var address = GetAddress();
+            var result = tools.CreateHttp(CheckContainerCrashed)
+                .OnClient(client =>
+            {
+                var api = new CodexApi(client);
+                api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
+                return Time.Wait(action(api));
+            });
+            return result;
         }
 
-        private IHttp LongHttp()
+        private IEndpoint GetEndpoint()
         {
-            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name);
+            return tools
+                .CreateHttp(CheckContainerCrashed)
+                .CreateEndpoint(GetAddress(), "/api/codex/v1/", Container.Name);
         }
 
         private Address GetAddress()

ProjectPlugins/CodexPlugin/CodexApiTypes.cs

@@ -1,194 +0,0 @@
-using Newtonsoft.Json;
-
-namespace CodexPlugin
-{
-    public class CodexDebugResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string[] addrs { get; set; } = new string[0];
-        public string repo { get; set; } = string.Empty;
-        public string spr { get; set; } = string.Empty;
-        public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
-        public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
-        public CodexDebugVersionResponse codex { get; set; } = new();
-        public CodexDebugTableResponse table { get; set; } = new();
-    }
-
-    public class CodexDebugFutures
-    {
-        public int futures { get; set; }
-    }
-
-    public class CodexDebugTableResponse
-    {
-        public CodexDebugTableNodeResponse localNode { get; set; } = new();
-        public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
-    }
-
-    public class CodexDebugTableNodeResponse
-    {
-        public string nodeId { get; set; } = string.Empty;
-        public string peerId { get; set; } = string.Empty;
-        public string record { get; set; } = string.Empty;
-        public string address { get; set; } = string.Empty;
-        public bool seen { get; set; }
-    }
-
-    public class EnginePeerResponse
-    {
-        public string peerId { get; set; } = string.Empty;
-        public EnginePeerContextResponse context { get; set; } = new();
-    }
-
-    public class EnginePeerContextResponse
-    {
-        public int blocks { get; set; } = 0;
-        public int peerWants { get; set; } = 0;
-        public int exchanged { get; set; } = 0;
-        public string lastExchange { get; set; } = string.Empty;
-    }
-
-    public class SwitchPeerResponse
-    {
-        public string peerId { get; set; } = string.Empty;
-        public string key { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugVersionResponse
-    {
-        public string version { get; set; } = string.Empty;
-        public string revision { get; set; } = string.Empty;
-
-        public bool IsValid()
-        {
-            return !string.IsNullOrEmpty(version) && !string.IsNullOrEmpty(revision);
-        }
-
-        public override string ToString()
-        {
-            return JsonConvert.SerializeObject(this);
-        }
-    }
-
-    public class CodexDebugPeerResponse
-    {
-        public bool IsPeerFound { get; set; }
-
-        public string peerId { get; set; } = string.Empty;
-        public long seqNo { get; set; }
-        public string[] addresses { get; set; } = Array.Empty<string>();
-    }
-
-    public class CodexDebugThresholdBreaches
-    {
-        public string[] breaches { get; set; } = Array.Empty<string>();
-    }
-
-    public class CodexSalesAvailabilityRequest
-    {
-        public string size { get; set; } = string.Empty;
-        public string duration { get; set; } = string.Empty;
-        public string minPrice { get; set; } = string.Empty;
-        public string maxCollateral { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesAvailabilityResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string size { get; set; } = string.Empty;
-        public string duration { get; set; } = string.Empty;
-        public string minPrice { get; set; } = string.Empty;
-        public string maxCollateral { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesRequestStorageRequest
-    {
-        public string duration { get; set; } = string.Empty;
-        public string proofProbability { get; set; } = string.Empty;
-        public string reward { get; set; } = string.Empty;
-        public string collateral { get; set; } = string.Empty;
-        public string? expiry { get; set; }
-        public uint? nodes { get; set; }
-        public uint? tolerance { get; set; }
-    }
-
-    public class CodexStoragePurchase
-    {
-        public string state { get; set; } = string.Empty;
-        public string error { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugBlockExchangeResponse
-    {
-        public CodexDebugBlockExchangeResponsePeer[] peers { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeer>();
-        public int taskQueue { get; set; }
-        public int pendingBlocks { get; set; }
-
-        public override string ToString()
-        {
-            if (peers.Length == 0 && taskQueue == 0 && pendingBlocks == 0) return "all-empty";
-
-            return $"taskqueue: {taskQueue} pendingblocks: {pendingBlocks} peers: {string.Join(",", peers.Select(p => p.ToString()))}";
-        }
-    }
-
-    public class CodexDebugBlockExchangeResponsePeer
-    {
-        public string peerid { get; set; } = string.Empty;
-        public CodexDebugBlockExchangeResponsePeerHasBlock[] hasBlocks { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeerHasBlock>();
-        public CodexDebugBlockExchangeResponsePeerWant[] wants { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeerWant>();
-        public int exchanged { get; set; }
-
-        public override string ToString()
-        {
-            return $"(blocks:{hasBlocks.Length} wants:{wants.Length})";
-        }
-    }
-
-    public class CodexDebugBlockExchangeResponsePeerHasBlock
-    {
-        public string cid { get; set; } = string.Empty;
-        public bool have { get; set; }
-        public string price { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugBlockExchangeResponsePeerWant
-    {
-        public string block { get; set; } = string.Empty;
-        public int priority { get; set; }
-        public bool cancel { get; set; }
-        public string wantType { get; set; } = string.Empty;
-        public bool sendDontHave { get; set; }
-    }
-
-    public class CodexDebugRepoStoreResponse
-    {
-        public string cid { get; set; } = string.Empty; 
-    }
-
-    public class CodexLocalData
-    {
-        public CodexLocalData(ContentId cid, CodexLocalDataManifestResponse manifest)
-        {
-            Cid = cid;
-            Manifest = manifest;
-        }
-
-        public ContentId Cid { get; }
-        public CodexLocalDataManifestResponse Manifest { get; }
-    }
-
-    public class CodexLocalDataResponse
-    {
-        public string cid { get; set; } = string.Empty;
-        public CodexLocalDataManifestResponse manifest { get; set; } = new();
-    }
-
-    public class CodexLocalDataManifestResponse
-    {
-        public string rootHash { get; set; } = string.Empty;
-        public int originalBytes { get; set; }
-        public int blockSize { get; set; }
-        public bool @protected { get; set; }
-    }
-}

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -7,9 +7,7 @@ namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter();
-
-        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-cd280d4-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";
@@ -29,6 +27,9 @@ namespace CodexPlugin
             SetResourcesRequest(milliCPUs: 100, memory: 100.MB());
             //SetResourceLimits(milliCPUs: 4000, memory: 12.GB());
 
+            SetSchedulingAffinity(notIn: "false");
+            SetSystemCriticalPriority();
+
             var config = startupConfig.Get<CodexStartupConfig>();
 
             var apiPort = CreateApiPort(config, ApiPortTag);
@@ -45,8 +46,9 @@ namespace CodexPlugin
 
             if (config.PublicTestNet != null)
             {
-                AddEnvVar("CODEX_NAT", config.PublicTestNet.PublicNatIP);
+                // This makes the node announce itself to its public IP address.
                 AddEnvVar("NAT_IP_AUTO", "false");
+                AddEnvVar("NAT_PUBLIC_IP_AUTO", PublicIpService.Address);
             }
             else
             {
@@ -101,15 +103,16 @@ namespace CodexPlugin
 
                 AddEnvVar("CODEX_ETH_PROVIDER", $"{wsAddress.Host.Replace("http://", "ws://")}:{wsAddress.Port}");
                 AddEnvVar("CODEX_MARKETPLACE_ADDRESS", marketplaceAddress);
-                AddEnvVar("CODEX_PERSISTENCE", "true");
+
+                var marketplaceSetup = config.MarketplaceConfig.MarketplaceSetup;
 
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
-                var mStart = marketplaceStarter.Start();
-                AddEnvVar("PRIV_KEY", mStart.PrivateKey);
-                Additional(mStart);
+                AddEnvVar("PRIV_KEY", marketplaceSetup.EthAccount.PrivateKey);
+                Additional(marketplaceSetup.EthAccount);
 
-                if (config.MarketplaceConfig.IsValidator)
+                SetCommandOverride(marketplaceSetup);
+                if (marketplaceSetup.IsValidator)
                 {
                    AddEnvVar("CODEX_VALIDATOR", "true");
                 }
@@ -121,6 +124,18 @@ namespace CodexPlugin
             }
         }
 
+        private void SetCommandOverride(MarketplaceSetup ms)
+        {
+            if (ms.IsStorageNode)
+            {
+                OverrideCommand("bash", "/docker-entrypoint.sh", "codex", "persistence", "prover");
+            }
+            else
+            {
+                OverrideCommand("bash", "/docker-entrypoint.sh", "codex", "persistence");
+            }
+        }
+
         private Port CreateApiPort(CodexStartupConfig config, string tag)
         {
             if (config.PublicTestNet == null) return AddExposedPort(tag);

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -6,8 +6,12 @@ namespace CodexPlugin
 {
     public class CodexDeployment
     {
-        public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment, CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer, RunningContainers? discordBotContainer, DeploymentMetadata metadata)
+        public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
+            CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer,
+            RunningContainers? discordBotContainer, DeploymentMetadata metadata,
+            String id)
         {
+            Id = id;
             CodexInstances = codexInstances;
             GethDeployment = gethDeployment;
             CodexContractsDeployment = codexContractsDeployment;
@@ -16,6 +20,7 @@ namespace CodexPlugin
             Metadata = metadata;
         }
 
+        public String Id { get; }
         public CodexInstance[] CodexInstances { get; }
         public GethDeployment GethDeployment { get; }
         public CodexContractsDeployment CodexContractsDeployment { get; }
@@ -26,19 +31,22 @@ namespace CodexPlugin
 
     public class CodexInstance
     {
-        public CodexInstance(RunningContainers containers, CodexDebugResponse info)
+        public CodexInstance(RunningContainers containers, DebugInfo info)
         {
             Containers = containers;
             Info = info;
         }
 
         public RunningContainers Containers { get; }
-        public CodexDebugResponse Info { get; }
+        public DebugInfo Info { get; }
     }
 
     public class DeploymentMetadata
     {
-        public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
+        public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace,
+            int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel,
+            int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI,
+            int blockMN)
         {
             Name = name;
             StartUtc = startUtc;
@@ -73,4 +81,4 @@ namespace CodexPlugin
         public int BlockMI { get; }
         public int BlockMN { get; }
     }
-}
+}

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -12,28 +12,26 @@ namespace CodexPlugin
     public interface ICodexNode : IHasContainer, IHasMetricsScrapeTarget, IHasEthAddress
     {
         string GetName();
-        CodexDebugResponse GetDebugInfo();
-        CodexDebugPeerResponse GetDebugPeer(string peerId);
-        // These debug methods are not available in master-line Codex. Use only for custom builds.
-        //CodexDebugBlockExchangeResponse GetDebugBlockExchange();
-        //CodexDebugRepoStoreResponse[] GetDebugRepoStore();
+        DebugInfo GetDebugInfo();
+        DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
-        CodexLocalData[] LocalFiles();
+        LocalDatasetList LocalFiles();
         void ConnectToPeer(ICodexNode node);
-        CodexDebugVersionResponse Version { get; }
+        DebugInfoVersion Version { get; }
         IMarketplaceAccess Marketplace { get; }
         CrashWatcher CrashWatcher { get; }
         PodInfo GetPodInfo();
-        void Stop();
+        ITransferSpeeds TransferSpeeds { get; }
+        void Stop(bool waitTillStopped);
     }
 
     public class CodexNode : ICodexNode
     {
-        private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
         private const string UploadFailedMessage = "Unable to store block";
         private readonly IPluginTools tools;
         private readonly EthAddress? ethAddress;
+        private readonly TransferSpeeds transferSpeeds;
 
         public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAddress? ethAddress)
         {
@@ -42,7 +40,8 @@ namespace CodexPlugin
             CodexAccess = codexAccess;
             Group = group;
             Marketplace = marketplaceAccess;
-            Version = new CodexDebugVersionResponse();
+            Version = new DebugInfoVersion();
+            transferSpeeds = new TransferSpeeds();
         }
 
         public RunningContainer Container { get { return CodexAccess.Container; } }
@@ -50,7 +49,9 @@ namespace CodexPlugin
         public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; }
         public CodexNodeGroup Group { get; }
         public IMarketplaceAccess Marketplace { get; }
-        public CodexDebugVersionResponse Version { get; private set; }
+        public DebugInfoVersion Version { get; private set; }
+        public ITransferSpeeds TransferSpeeds { get => transferSpeeds; }
+
         public IMetricsScrapeTarget MetricsScrapeTarget
         {
             get
@@ -58,6 +59,7 @@ namespace CodexPlugin
                 return new MetricsScrapeTarget(CodexAccess.Container, CodexContainerRecipe.MetricsPortTag);
             }
         }
+
         public EthAddress EthAddress 
         {
             get
@@ -72,40 +74,33 @@ namespace CodexPlugin
             return CodexAccess.Container.Name;
         }
 
-        public CodexDebugResponse GetDebugInfo()
+        public DebugInfo GetDebugInfo()
         {
             var debugInfo = CodexAccess.GetDebugInfo();
-            var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
-            Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
+            var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
+            Log($"Got DebugInfo with id: '{debugInfo.Id}'. This node knows: {known}");
             return debugInfo;
         }
 
-        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        public DebugPeer GetDebugPeer(string peerId)
         {
             return CodexAccess.GetDebugPeer(peerId);
         }
 
-        public CodexDebugBlockExchangeResponse GetDebugBlockExchange()
-        {
-            return CodexAccess.GetDebugBlockExchange();
-        }
-
-        public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
-        {
-            return CodexAccess.GetDebugRepoStore();
-        }
-
         public ContentId UploadFile(TrackedFile file)
         {
             using var fileStream = File.OpenRead(file.Filename);
 
             var logMessage = $"Uploading file {file.Describe()}...";
             Log(logMessage);
-            var response = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
+            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
             {
                 return CodexAccess.UploadFile(fileStream);
             });
 
+            var response = measurement.Value;
+            transferSpeeds.AddUploadSample(file.GetFilesize(), measurement.Duration);
+
             if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
             if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
 
@@ -118,14 +113,15 @@ namespace CodexPlugin
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
             Log(logMessage);
             var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
-            Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file));
+            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file));
+            transferSpeeds.AddDownloadSample(file.GetFilesize(), measurement);
             Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
             return file;
         }
 
-        public CodexLocalData[] LocalFiles()
+        public LocalDatasetList LocalFiles()
         {
-            return CodexAccess.LocalFiles().Select(l => new CodexLocalData(new ContentId(l.cid), l.manifest)).ToArray();
+            return CodexAccess.LocalFiles();
         }
 
         public void ConnectToPeer(ICodexNode node)
@@ -134,9 +130,8 @@ namespace CodexPlugin
 
             Log($"Connecting to peer {peer.GetName()}...");
             var peerInfo = node.GetDebugInfo();
-            var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
+            CodexAccess.ConnectToPeer(peerInfo.Id, GetPeerMultiAddresses(peer, peerInfo));
 
-            FrameworkAssert.That(response == SuccessfullyConnectedMessage, "Unable to connect codex nodes.");
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
@@ -145,43 +140,42 @@ namespace CodexPlugin
             return CodexAccess.GetPodInfo();
         }
 
-        public void Stop()
+        public void Stop(bool waitTillStopped)
         {
             if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
                 "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
                 "available for codex-nodes in groups of 1.");
 
-            Group.BringOffline();
+            Group.BringOffline(waitTillStopped);
         }
 
         public void EnsureOnlineGetVersionResponse()
         {
             var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
-            var nodePeerId = debugInfo.id;
+            var nodePeerId = debugInfo.Id;
             var nodeName = CodexAccess.Container.Name;
 
-            if (!debugInfo.codex.IsValid())
+            if (!debugInfo.Version.IsValid())
             {
-                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}");
+                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
             }
 
             var log = tools.GetLog();
             log.AddStringReplace(nodePeerId, nodeName);
-            log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
-            Version = debugInfo.codex;
+            log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName);
+            Version = debugInfo.Version;
         }
 
-        private string GetPeerMultiAddress(CodexNode peer, CodexDebugResponse peerInfo)
+        private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)
         {
-            var multiAddress = peerInfo.addrs.First();
-            // Todo: Is there a case where First address in list is not the way?
-
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
             var workflow = tools.CreateWorkflow();
             var podInfo = workflow.GetPodInfo(peer.Container);
 
-            return multiAddress.Replace("0.0.0.0", podInfo.Ip);
+            return peerInfo.Addrs.Select(a => a
+                .Replace("0.0.0.0", podInfo.Ip))
+                .ToArray();
         }
 
         private void DownloadToFile(string contentId, TrackedFile file)
@@ -204,24 +198,4 @@ namespace CodexPlugin
             tools.GetLog().Log($"{GetName()}: {msg}");
         }
     }
-
-    public class ContentId
-    {
-        public ContentId(string id)
-        {
-            Id = id;
-        }
-
-        public string Id { get; }
-
-        public override bool Equals(object? obj)
-        {
-            return obj is ContentId id && Id == id.Id;
-        }
-
-        public override int GetHashCode()
-        {
-            return HashCode.Combine(Id);
-        }
-    }
 }

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -35,9 +35,9 @@ namespace CodexPlugin
 
         private EthAddress? GetEthAddress(CodexAccess access)
         {
-            var mStart = access.Container.Recipe.Additionals.Get<MarketplaceStartResults>();
-            if (mStart == null) return null;
-            return mStart.EthAddress;
+            var ethAccount = access.Container.Recipe.Additionals.Get<EthAccount>();
+            if (ethAccount == null) return null;
+            return ethAccount.EthAddress;
         }
 
         public CrashWatcher CreateCrashWatcher(RunningContainer c)

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -7,7 +7,7 @@ namespace CodexPlugin
 {
     public interface ICodexNodeGroup : IEnumerable<ICodexNode>, IHasManyMetricScrapeTargets
     {
-        void BringOffline();
+        void BringOffline(bool waitTillStopped);
         ICodexNode this[int index] { get; }
     }
 
@@ -20,7 +20,7 @@ namespace CodexPlugin
             this.starter = starter;
             Containers = containers;
             Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
-            Version = new CodexDebugVersionResponse();
+            Version = new DebugInfoVersion();
         }
 
         public ICodexNode this[int index]
@@ -31,9 +31,9 @@ namespace CodexPlugin
             }
         }
 
-        public void BringOffline()
+        public void BringOffline(bool waitTillStopped)
         {
-            starter.BringOffline(this);
+            starter.BringOffline(this, waitTillStopped);
             // Clear everything. Prevent accidental use.
             Nodes = Array.Empty<CodexNode>();
             Containers = null!;
@@ -41,7 +41,7 @@ namespace CodexPlugin
 
         public RunningContainers[] Containers { get; private set; }
         public CodexNode[] Nodes { get; private set; }
-        public CodexDebugVersionResponse Version { get; private set; }
+        public DebugInfoVersion Version { get; private set; }
         public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray();
 
         public IEnumerator<ICodexNode> GetEnumerator()
@@ -65,7 +65,7 @@ namespace CodexPlugin
             var versionResponses = Nodes.Select(n => n.Version);
 
             var first = versionResponses.First();
-            if (!versionResponses.All(v => v.version == first.version && v.revision == first.revision))
+            if (!versionResponses.All(v => v.Version == first.Version && v.Revision == first.Revision))
             {
                 throw new Exception("Inconsistent version information received from one or more Codex nodes: " +
                     string.Join(",", versionResponses.Select(v => v.ToString())));

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -1,4 +1,4 @@
-using Core;
+using Core;
 using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
@@ -19,12 +19,13 @@ namespace CodexPlugin
 
         public void Announce()
         {
-            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}'");
+            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
         }
 
         public void AddMetadata(IAddMetadata metadata)
         {
             metadata.Add("codexid", codexStarter.GetCodexId());
+            metadata.Add("codexrevision", codexStarter.GetCodexRevision());
         }
 
         public void Decommission()
@@ -51,8 +52,8 @@ namespace CodexPlugin
             var mconfig = codexSetup.MarketplaceConfig;
             foreach (var node in result)
             {
-                mconfig.GethNode.SendEth(node, mconfig.InitialEth);
-                mconfig.CodexContracts.MintTestTokens(node, mconfig.InitialTokens);
+                mconfig.GethNode.SendEth(node, mconfig.MarketplaceSetup.InitialEth);
+                mconfig.CodexContracts.MintTestTokens(node, mconfig.MarketplaceSetup.InitialTestTokens);
             }
         }
 

ProjectPlugins/CodexPlugin/CodexPlugin.csproj

@@ -7,7 +7,23 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <None Remove="openapi.yaml" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <OpenApiReference Include="openapi.yaml" CodeGenerator="NSwagCSharp" Namespace="CodexOpenApi" ClassName="CodexApi" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.ApiDescription.Client" Version="7.0.2">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="NSwag.ApiDescription.Client" Version="13.18.2">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>
@@ -18,4 +34,8 @@
     <ProjectReference Include="..\MetricsPlugin\MetricsPlugin.csproj" />
   </ItemGroup>
 
+  <Target Name="PreBuild" BeforeTargets="PreBuildEvent">
+    <Exec Command="dotnet run --project $(ProjectDir)\..\CodexPluginPrebuild" />
+  </Target>
+
 </Project>

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -17,7 +17,7 @@ namespace CodexPlugin
         ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration);
         ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks);
         ICodexSetup EnableMetrics();
-        ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens, bool isValidator = false);
+        ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Action<IMarketplaceSetup> marketplaceSetup);
         /// <summary>
         /// Provides an invalid proof every N proofs
         /// </summary>
@@ -25,8 +25,23 @@ namespace CodexPlugin
         ICodexSetup AsPublicTestNet(CodexTestNetConfig testNetConfig);
     }
 
+    public interface IMarketplaceSetup
+    {
+        IMarketplaceSetup WithInitial(Ether eth, TestToken tokens);
+        IMarketplaceSetup WithAccount(EthAccount account);
+        IMarketplaceSetup AsStorageNode();
+        IMarketplaceSetup AsValidator();
+    }
+
     public class CodexLogCustomTopics
     {
+        public CodexLogCustomTopics(CodexLogLevel discV5, CodexLogLevel libp2p, CodexLogLevel blockExchange)
+        {
+            DiscV5 = discV5;
+            Libp2p = libp2p;
+            BlockExchange = blockExchange;
+        }
+
         public CodexLogCustomTopics(CodexLogLevel discV5, CodexLogLevel libp2p)
         {
             DiscV5 = discV5;
@@ -35,6 +50,7 @@ namespace CodexPlugin
 
         public CodexLogLevel DiscV5 { get; set; }
         public CodexLogLevel Libp2p { get; set; }
+        public CodexLogLevel? BlockExchange { get; }
     }
 
     public class CodexSetup : CodexStartupConfig, ICodexSetup
@@ -60,7 +76,7 @@ namespace CodexPlugin
 
         public ICodexSetup WithBootstrapNode(ICodexNode node)
         {
-            BootstrapSpr = node.GetDebugInfo().spr;
+            BootstrapSpr = node.GetDebugInfo().Spr;
             return this;
         }
 
@@ -107,9 +123,12 @@ namespace CodexPlugin
             return this;
         }
 
-        public ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens, bool isValidator = false)
+        public ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Action<IMarketplaceSetup> marketplaceSetup)
         {
-            MarketplaceConfig = new MarketplaceInitialConfig(gethNode, codexContracts, initialEth, initialTokens, isValidator);
+            var ms = new MarketplaceSetup();
+            marketplaceSetup(ms);
+
+            MarketplaceConfig = new MarketplaceInitialConfig(ms, gethNode, codexContracts);
             return this;
         }
 
@@ -133,12 +152,57 @@ namespace CodexPlugin
 
         private IEnumerable<string> DescribeArgs()
         {
-            if (PublicTestNet != null) yield return $"<!>Public TestNet at {PublicTestNet.PublicNatIP}:{PublicTestNet.PublicListenPort}<!>";
+            if (PublicTestNet != null) yield return $"<!>Public TestNet with listenPort: {PublicTestNet.PublicListenPort}<!>";
             yield return $"LogLevel={LogLevelWithTopics()}";
             if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
             if (StorageQuota != null) yield return $"StorageQuota={StorageQuota}";
             if (SimulateProofFailures != null) yield return $"SimulateProofFailures={SimulateProofFailures}";
-            if (MarketplaceConfig != null) yield return $"IsValidator={MarketplaceConfig.IsValidator}";
+            if (MarketplaceConfig != null) yield return $"MarketplaceSetup={MarketplaceConfig.MarketplaceSetup}";
+        }
+    }
+
+    public class MarketplaceSetup : IMarketplaceSetup
+    {
+        public bool IsStorageNode { get; private set; }
+        public bool IsValidator { get; private set; }
+        public Ether InitialEth { get; private set; } = 0.Eth();
+        public TestToken InitialTestTokens { get; private set; } = 0.TestTokens();
+        public EthAccount EthAccount { get; private set; } = EthAccount.GenerateNew();
+
+        public IMarketplaceSetup AsStorageNode()
+        {
+            IsStorageNode = true;
+            return this;
+        }
+
+        public IMarketplaceSetup AsValidator()
+        {
+            IsValidator = true;
+            return this;
+        }
+
+        public IMarketplaceSetup WithAccount(EthAccount account)
+        {
+            EthAccount = account;
+            return this;
+        }
+
+        public IMarketplaceSetup WithInitial(Ether eth, TestToken tokens)
+        {
+            InitialEth = eth;
+            InitialTestTokens = tokens;
+            return this;
+        }
+
+        public override string ToString()
+        {
+            var result = "[(clientNode)"; // When marketplace is enabled, being a clientNode is implicit.
+            result += IsStorageNode ? "(storageNode)" : "()";
+            result += IsValidator ? "(validator)" : "() ";
+            result += $"Address: '{EthAccount.EthAddress}' ";
+            result += $"InitialEth/TT({InitialEth.Eth}/{InitialTestTokens.Amount})";
+            result += "] ";
+            return result;
         }
     }
 }

ProjectPlugins/CodexPlugin/CodexStarter.cs

@@ -9,11 +9,14 @@ namespace CodexPlugin
     {
         private readonly IPluginTools pluginTools;
         private readonly CodexContainerRecipe recipe = new CodexContainerRecipe();
-        private CodexDebugVersionResponse? versionResponse;
+        private readonly ApiChecker apiChecker;
+        private DebugInfoVersion? versionResponse;
 
         public CodexStarter(IPluginTools pluginTools)
         {
             this.pluginTools = pluginTools;
+
+            apiChecker = new ApiChecker(pluginTools);
         }
 
         public RunningContainers[] BringOnline(CodexSetup codexSetup)
@@ -25,6 +28,8 @@ namespace CodexPlugin
 
             var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
 
+            apiChecker.CheckCompatibility(containers);
+
             foreach (var rc in containers)
             {
                 var podInfo = GetPodInfo(rc);
@@ -48,24 +53,30 @@ namespace CodexPlugin
             return group;
         }
 
-        public void BringOffline(CodexNodeGroup group)
+        public void BringOffline(CodexNodeGroup group, bool waitTillStopped)
         {
             Log($"Stopping {group.Describe()}...");
             StopCrashWatcher(group);
             var workflow = pluginTools.CreateWorkflow();
             foreach (var c in group.Containers)
             {
-                workflow.Stop(c);
+                workflow.Stop(c, waitTillStopped);
             }
             Log("Stopped.");
         }
 
         public string GetCodexId()
         {
-            if (versionResponse != null) return versionResponse.version;
+            if (versionResponse != null) return versionResponse.Version;
             return recipe.Image;
         }
 
+        public string GetCodexRevision()
+        {
+            if (versionResponse != null) return versionResponse.Revision;
+            return "unknown";
+        }
+
         private StartupConfig CreateStartupConfig(CodexSetup codexSetup)
         {
             var startupConfig = new StartupConfig();

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -52,12 +52,38 @@ namespace CodexPlugin
                     "connection",
                     "connmanager",
                     "websock",
-                    "ws-session"
+                    "ws-session",
+                    "dialer",
+                    "muxedupgrade",
+                    "upgrade",
+                    "identify"
+                };
+                var blockExchangeTopics = new[]
+                {
+                    "codex",
+                    "pendingblocks",
+                    "peerctxstore",
+                    "discoveryengine",
+                    "blockexcengine",
+                    "blockexcnetwork",
+                    "blockexcnetworkpeer"
+                };
+                var contractClockTopics = new[]
+                {
+                    "contracts",
+                    "clock"
                 };
 
                 level = $"{level};" +
                     $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" +
-                    $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)}";
+                    $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)};" +
+                    // Contract clock is always set to warn. It logs a trace every second.
+                    $"{CodexLogLevel.Warn.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)}";
+
+                if (CustomTopics.BlockExchange != null)
+                {
+                    level += $";{CustomTopics.BlockExchange.ToString()!.ToLowerInvariant()}:{string.Join(",", blockExchangeTopics)}";
+                }
             }
             return level;
         }
@@ -65,7 +91,6 @@ namespace CodexPlugin
 
     public class CodexTestNetConfig
     {
-        public string PublicNatIP { get; set; } = string.Empty;
         public int PublicDiscoveryPort { get; set; }
         public int PublicListenPort { get; set; }
     }

ProjectPlugins/CodexPlugin/CodexTypes.cs

@@ -0,0 +1,108 @@
+using Newtonsoft.Json;
+using Utils;
+
+namespace CodexPlugin
+{
+    public class DebugInfo
+    {
+        public string[] Addrs { get; set; } = Array.Empty<string>();
+        public string Spr { get; set; } = string.Empty;
+        public string Id { get; set; } = string.Empty;
+        public string[] AnnounceAddresses { get; set; } = Array.Empty<string>();
+        public DebugInfoVersion Version { get; set; } = new();
+        public DebugInfoTable Table { get; set; } = new();
+    }
+
+    public class DebugInfoVersion
+    {
+        public string Version { get; set; } = string.Empty;
+        public string Revision { get; set; } = string.Empty;
+
+        public bool IsValid()
+        {
+            return !string.IsNullOrEmpty(Version) && !string.IsNullOrEmpty(Revision);
+        }
+
+        public override string ToString()
+        {
+            return JsonConvert.SerializeObject(this);
+        }
+    }
+
+    public class DebugInfoTable
+    {
+        public DebugInfoTableNode LocalNode { get; set; } = new();
+        public DebugInfoTableNode[] Nodes { get; set; } = Array.Empty<DebugInfoTableNode>();
+    }
+
+    public class DebugInfoTableNode
+    {
+        public string NodeId { get; set; } = string.Empty;
+        public string PeerId { get; set; } = string.Empty;
+        public string Record { get; set; } = string.Empty;
+        public string Address { get; set; } = string.Empty;
+        public bool Seen { get; set; }
+    }
+
+    public class DebugPeer
+    {
+        public bool IsPeerFound { get; set; }
+        public string PeerId { get; set; } = string.Empty;
+        public string[] Addresses { get; set; } = Array.Empty<string>();
+    }
+
+    public class LocalDatasetList
+    {
+        public LocalDataset[] Content { get; set; } = Array.Empty<LocalDataset>();
+    } 
+
+    public class LocalDataset
+    {
+        public ContentId Cid { get; set; } = new();
+        public Manifest Manifest { get; set; } = new();
+    }
+
+    public class Manifest
+    {
+        public string RootHash { get; set; } = string.Empty;
+        public ByteSize OriginalBytes { get; set; } = ByteSize.Zero;
+        public ByteSize BlockSize { get; set; } = ByteSize.Zero;
+        public bool Protected { get; set; }
+    }
+
+    public class SalesRequestStorageRequest
+    {
+        public string Duration { get; set; } = string.Empty;
+        public string ProofProbability { get; set; } = string.Empty;
+        public string Reward { get; set; } = string.Empty;
+        public string Collateral { get; set; } = string.Empty;
+        public string? Expiry { get; set; }
+        public uint? Nodes { get; set; }
+        public uint? Tolerance { get; set; }
+    }
+
+    public class ContentId
+    {
+        public ContentId()
+        {
+            Id = string.Empty;
+        }
+
+        public ContentId(string id)
+        {
+            Id = id;
+        }
+
+        public string Id { get; }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is ContentId id && Id == id.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
+    }
+}

ProjectPlugins/CodexPlugin/Mapper.cs

@@ -0,0 +1,172 @@
+using CodexContractsPlugin;
+using Newtonsoft.Json.Linq;
+using System.Numerics;
+using Utils;
+
+namespace CodexPlugin
+{
+    public class Mapper
+    {
+        public DebugInfo Map(CodexOpenApi.DebugInfo debugInfo)
+        {
+            return new DebugInfo
+            {
+                Id = debugInfo.Id,
+                Spr = debugInfo.Spr,
+                Addrs = debugInfo.Addrs.ToArray(),
+                AnnounceAddresses = JArray(debugInfo.AdditionalProperties, "announceAddresses").Select(x => x.ToString()).ToArray(),
+                Version = MapDebugInfoVersion(JObject(debugInfo.AdditionalProperties, "codex")),
+                Table = MapDebugInfoTable(JObject(debugInfo.AdditionalProperties, "table"))
+            };
+        }
+
+        public LocalDatasetList Map(CodexOpenApi.DataList dataList)
+        {
+            return new LocalDatasetList
+            {
+                Content = dataList.Content.Select(Map).ToArray()
+            };
+        }
+
+        public LocalDataset Map(CodexOpenApi.DataItem dataItem)
+        {
+            return new LocalDataset
+            {
+                Cid = new ContentId(dataItem.Cid),
+                Manifest = MapManifest(dataItem.Manifest)
+            };
+        }
+
+        public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability)
+        {
+            return new CodexOpenApi.SalesAvailabilityCREATE
+            {
+                Duration = ToDecInt(availability.MaxDuration.TotalSeconds),
+                MinPrice = ToDecInt(availability.MinPriceForTotalSpace),
+                MaxCollateral = ToDecInt(availability.MaxCollateral),
+                TotalSize = ToDecInt(availability.TotalSpace.SizeInBytes)
+            };
+        }
+
+        public CodexOpenApi.StorageRequestCreation Map(StoragePurchaseRequest purchase)
+        {
+            return new CodexOpenApi.StorageRequestCreation
+            {
+                Duration = ToDecInt(purchase.Duration.TotalSeconds),
+                ProofProbability = ToDecInt(purchase.ProofProbability),
+                Reward = ToDecInt(purchase.PricePerSlotPerSecond),
+                Collateral = ToDecInt(purchase.RequiredCollateral),
+                Expiry = ToDecInt(DateTimeOffset.UtcNow.ToUnixTimeSeconds() + purchase.Expiry.TotalSeconds),
+                Nodes = purchase.MinRequiredNumberOfNodes,
+                Tolerance = purchase.NodeFailureTolerance
+            };
+        }
+
+        public StoragePurchase Map(CodexOpenApi.Purchase purchase)
+        {
+            return new StoragePurchase
+            {
+                State = purchase.State,
+                Error = purchase.Error
+            };
+        }
+
+        public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD read)
+        {
+            return new StorageAvailability(
+                totalSpace: new Utils.ByteSize(Convert.ToInt64(read.TotalSize)),
+                maxDuration: TimeSpan.FromSeconds(Convert.ToDouble(read.Duration)),
+                minPriceForTotalSpace: new TestToken(Convert.ToDecimal(read.MinPrice)),
+                maxCollateral: new TestToken(Convert.ToDecimal(read.MaxCollateral))
+            )
+            {
+                Id = read.Id
+            };
+        }
+
+        private DebugInfoVersion MapDebugInfoVersion(JObject obj)
+        {
+            return new DebugInfoVersion
+            {
+                Version = StringOrEmpty(obj, "version"),
+                Revision = StringOrEmpty(obj, "revision")
+            };
+        }
+
+        private DebugInfoTable MapDebugInfoTable(JObject obj)
+        {
+            return new DebugInfoTable
+            {
+                LocalNode = MapDebugInfoTableNode(obj.GetValue("localNode")),
+                Nodes = new DebugInfoTableNode[0]
+            };
+        }
+
+        private DebugInfoTableNode MapDebugInfoTableNode(JToken? token)
+        {
+            var obj = token as JObject;
+            if (obj == null) return new DebugInfoTableNode();
+
+            return new DebugInfoTableNode
+            {
+                Address = StringOrEmpty(obj, "address"),
+                NodeId = StringOrEmpty(obj, "nodeId"),
+                PeerId = StringOrEmpty(obj, "peerId"),
+                Record = StringOrEmpty(obj, "record"),
+                Seen = Bool(obj, "seen")
+            };
+        }
+
+        private Manifest MapManifest(CodexOpenApi.ManifestItem manifest)
+        {
+            return new Manifest
+            {
+                BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)),
+                OriginalBytes = new ByteSize(Convert.ToInt64(manifest.OriginalBytes)),
+                RootHash = manifest.RootHash,
+                Protected = manifest.Protected
+            };
+        }
+
+        private JArray JArray(IDictionary<string, object> map, string name)
+        {
+            return (JArray)map[name];
+        }
+
+        private JObject JObject(IDictionary<string, object> map, string name)
+        {
+            return (JObject)map[name];
+        }
+
+        private string StringOrEmpty(JObject obj, string name)
+        {
+            if (obj.TryGetValue(name, out var token))
+            {
+                var str = (string?)token;
+                if (!string.IsNullOrEmpty(str)) return str;
+            }
+            return string.Empty;
+        }
+
+        private bool Bool(JObject obj, string name)
+        {
+            if (obj.TryGetValue(name, out var token))
+            {
+                return (bool)token;
+            }
+            return false;
+        }
+
+        private string ToDecInt(double d)
+        {
+            var i = new BigInteger(d);
+            return i.ToString("D");
+        }
+
+        private string ToDecInt(TestToken t)
+        {
+            var i = new BigInteger(t.Amount);
+            return i.ToString("D");
+        }
+    }
+}

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -1,15 +1,13 @@
-using CodexContractsPlugin;
-using Logging;
+using Logging;
 using Newtonsoft.Json;
 using Utils;
-using System.Numerics;
 
 namespace CodexPlugin
 {
     public interface IMarketplaceAccess
     {
-        string MakeStorageAvailable(ByteSize size, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration);
-        StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
+        string MakeStorageAvailable(StorageAvailability availability);
+        StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase);
     }
 
     public class MarketplaceAccess : IMarketplaceAccess
@@ -23,71 +21,35 @@ namespace CodexPlugin
             this.codexAccess = codexAccess;
         }
 
-        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        public StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
         {
-            var request = new CodexSalesRequestStorageRequest
-            {
-                duration = ToDecInt(duration.TotalSeconds),
-                proofProbability = ToDecInt(proofProbability),
-                reward = ToDecInt(pricePerSlotPerSecond),
-                collateral = ToDecInt(requiredCollateral),
-                expiry = null,
-                nodes = minRequiredNumberOfNodes,
-                tolerance = null,
-            };
+            purchase.Log(log);
 
-            Log($"Requesting storage for: {contentId.Id}... (" +
-                $"pricePerSlotPerSecond: {pricePerSlotPerSecond}, " +
-                $"requiredCollateral: {requiredCollateral}, " +
-                $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
-                $"proofProbability: {proofProbability}, " +
-                $"duration: {Time.FormatDuration(duration)})");
+            var response = codexAccess.RequestStorage(purchase);
 
-            var response = codexAccess.RequestStorage(request, contentId.Id);
-
-            if (response == "Purchasing not available")
+            if (string.IsNullOrEmpty(response) ||
+                response == "Purchasing not available" ||
+                response == "Expiry required" ||
+                response == "Expiry needs to be in future" ||
+                response == "Expiry has to be before the request's end (now + duration)")
             {
                 throw new InvalidOperationException(response);
             }
 
             Log($"Storage requested successfully. PurchaseId: '{response}'.");
 
-            return new StoragePurchaseContract(log, codexAccess, response, duration);
+            return new StoragePurchaseContract(log, codexAccess, response, purchase);
         }
 
-        public string MakeStorageAvailable(ByteSize totalSpace, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration)
+        public string MakeStorageAvailable(StorageAvailability availability)
         {
-            var request = new CodexSalesAvailabilityRequest
-            {
-                size = ToDecInt(totalSpace.SizeInBytes),
-                duration = ToDecInt(maxDuration.TotalSeconds),
-                maxCollateral = ToDecInt(maxCollateral),
-                minPrice = ToDecInt(minPriceForTotalSpace)
-            };
+            availability.Log(log);
 
-            Log($"Making storage available... (" +
-                $"size: {totalSpace}, " +
-                $"minPriceForTotalSpace: {minPriceForTotalSpace}, " +
-                $"maxCollateral: {maxCollateral}, " +
-                $"maxDuration: {Time.FormatDuration(maxDuration)})");
+            var response = codexAccess.SalesAvailability(availability);
 
-            var response = codexAccess.SalesAvailability(request);
+            Log($"Storage successfully made available. Id: {response.Id}");
 
-            Log($"Storage successfully made available. Id: {response.id}");
-
-            return response.id;
-        }
-
-        private string ToDecInt(double d)
-        {
-            var i = new BigInteger(d);
-            return i.ToString("D");
-        }
-
-        public string ToDecInt(TestToken t)
-        {
-            var i = new BigInteger(t.Amount);
-            return i.ToString("D");
+            return response.Id;
         }
 
         private void Log(string msg)
@@ -98,16 +60,16 @@ namespace CodexPlugin
 
     public class MarketplaceUnavailable : IMarketplaceAccess
     {
-        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        public string MakeStorageAvailable(StorageAvailability availability)
         {
             Unavailable();
-            return null!;
+            throw new NotImplementedException();
         }
 
-        public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration)
+        public StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase)
         {
             Unavailable();
-            return string.Empty;
+            throw new NotImplementedException();
         }
 
         private void Unavailable()
@@ -121,22 +83,26 @@ namespace CodexPlugin
     {
         private readonly ILog log;
         private readonly CodexAccess codexAccess;
+        private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(10);
         private DateTime? contractStartUtc;
 
-        public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, TimeSpan contractDuration)
+        public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase)
         {
             this.log = log;
             this.codexAccess = codexAccess;
             PurchaseId = purchaseId;
-            ContractDuration = contractDuration;
+            Purchase = purchase;
         }
 
         public string PurchaseId { get; }
-        public TimeSpan ContractDuration { get; }
+        public StoragePurchaseRequest Purchase { get; }
 
         public void WaitForStorageContractStarted()
         {
-            WaitForStorageContractStarted(TimeSpan.FromSeconds(30));
+            var timeout = Purchase.Expiry + gracePeriod;
+
+            WaitForStorageContractState(timeout, "started");
+            contractStartUtc = DateTime.UtcNow;
         }
 
         public void WaitForStorageContractFinished()
@@ -145,39 +111,14 @@ namespace CodexPlugin
             {
                 WaitForStorageContractStarted();
             }
-            var gracePeriod = TimeSpan.FromSeconds(10);
             var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
-            var timeout = (ContractDuration - currentContractTime) + gracePeriod;
+            var timeout = (Purchase.Duration - currentContractTime) + gracePeriod;
             WaitForStorageContractState(timeout, "finished");
         }
 
-        public void WaitForStorageContractFinished(ByteSize contractFileSize)
+        public StoragePurchase GetPurchaseStatus(string purchaseId)
         {
-            if (!contractStartUtc.HasValue)
-            {
-                WaitForStorageContractStarted(contractFileSize.ToTimeSpan());
-            }
-            var gracePeriod = TimeSpan.FromSeconds(10);
-            var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
-            var timeout = (ContractDuration - currentContractTime) + gracePeriod;
-            WaitForStorageContractState(timeout, "finished");
-        }
-
-        /// <summary>
-        /// Wait for contract to start. Max timeout depends on contract filesize. Allows more time for larger files.
-        /// </summary>
-        public void WaitForStorageContractStarted(ByteSize contractFileSize)
-        {
-            var filesizeInMb = contractFileSize.SizeInBytes / (1024 * 1024);
-            var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0);
-
-            WaitForStorageContractStarted(maxWaitTime);
-        }
-
-        public void WaitForStorageContractStarted(TimeSpan timeout)
-        {
-            WaitForStorageContractState(timeout, "started");
-            contractStartUtc = DateTime.UtcNow;
+            return codexAccess.GetPurchaseStatus(purchaseId);
         }
 
         private void WaitForStorageContractState(TimeSpan timeout, string desiredState)
@@ -190,9 +131,9 @@ namespace CodexPlugin
             {
                 var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
                 var statusJson = JsonConvert.SerializeObject(purchaseStatus);
-                if (purchaseStatus != null && purchaseStatus.state != lastState)
+                if (purchaseStatus != null && purchaseStatus.State != lastState)
                 {
-                    lastState = purchaseStatus.state;
+                    lastState = purchaseStatus.State;
                     log.Debug("Purchase status: " + statusJson);
                 }
 
@@ -210,10 +151,5 @@ namespace CodexPlugin
             }
             log.Log($"Contract '{desiredState}'.");
         }
-
-        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
-        {
-            return codexAccess.GetPurchaseStatus(purchaseId);
-        }
     }
 }

ProjectPlugins/CodexPlugin/MarketplaceInitialConfig.cs

@@ -5,19 +5,15 @@ namespace CodexPlugin
 {
     public class MarketplaceInitialConfig
     {
-        public MarketplaceInitialConfig(IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens, bool isValidator)
+        public MarketplaceInitialConfig(MarketplaceSetup marketplaceSetup, IGethNode gethNode, ICodexContracts codexContracts)
         {
+            MarketplaceSetup = marketplaceSetup;
             GethNode = gethNode;
             CodexContracts = codexContracts;
-            InitialEth = initialEth;
-            InitialTokens = initialTokens;
-            IsValidator = isValidator;
         }
 
+        public MarketplaceSetup MarketplaceSetup { get; }
         public IGethNode GethNode { get; }
         public ICodexContracts CodexContracts { get; }
-        public Ether InitialEth { get; }
-        public TestToken InitialTokens { get; }
-        public bool IsValidator { get; }
     }
 }

ProjectPlugins/CodexPlugin/MarketplaceStartResults.cs

@@ -1,17 +0,0 @@
-using GethPlugin;
-
-namespace CodexPlugin
-{
-    [Serializable]
-    public class MarketplaceStartResults
-    {
-        public MarketplaceStartResults(EthAddress ethAddress, string privateKey)
-        {
-            EthAddress = ethAddress;
-            PrivateKey = privateKey;
-        }
-
-        public EthAddress EthAddress { get; }
-        public string PrivateKey { get; }
-    }
-}

ProjectPlugins/CodexPlugin/MarketplaceStarter.cs

@@ -1,19 +0,0 @@
-using GethPlugin;
-using Nethereum.Hex.HexConvertors.Extensions;
-using Nethereum.Web3.Accounts;
-
-namespace CodexPlugin
-{
-    public class MarketplaceStarter
-    {
-        public MarketplaceStartResults Start()
-        {
-            var ecKey = Nethereum.Signer.EthECKey.GenerateKey();
-            var privateKey = ecKey.GetPrivateKeyAsBytes().ToHex();
-            var account = new Account(privateKey);
-            var ethAddress = new EthAddress(account.Address);
-
-            return new MarketplaceStartResults(ethAddress, account.PrivateKey);
-        }
-    }
-}

ProjectPlugins/CodexPlugin/MarketplaceTypes.cs

@@ -0,0 +1,67 @@
+using CodexContractsPlugin;
+using Logging;
+using Utils;
+
+namespace CodexPlugin
+{
+    public class StoragePurchaseRequest
+    {
+        public StoragePurchaseRequest(ContentId cid)
+        {
+            ContentId = cid;
+        }
+
+        public ContentId ContentId { get; set; }
+        public TestToken PricePerSlotPerSecond { get; set; } = 1.TestTokens();
+        public TestToken RequiredCollateral { get; set; } = 1.TestTokens();
+        public uint MinRequiredNumberOfNodes { get; set; }
+        public uint NodeFailureTolerance { get; set; }
+        public int ProofProbability { get; set; }
+        public TimeSpan Duration { get; set; }
+        public TimeSpan Expiry { get; set; }
+
+        public void Log(ILog log)
+        {
+            log.Log($"Requesting storage for: {ContentId.Id}... (" +
+                $"pricePerSlotPerSecond: {PricePerSlotPerSecond}, " +
+                $"requiredCollateral: {RequiredCollateral}, " +
+                $"minRequiredNumberOfNodes: {MinRequiredNumberOfNodes}, " +
+                $"nodeFailureTolerance: {NodeFailureTolerance}, " +
+                $"proofProbability: {ProofProbability}, " +
+                $"expiry: {Time.FormatDuration(Expiry)}, " +
+                $"duration: {Time.FormatDuration(Duration)})");
+        }
+    }
+
+    public class StoragePurchase
+    {
+        public string State { get; set; } = string.Empty;
+        public string Error { get; set; } = string.Empty;
+    }
+
+    public class StorageAvailability
+    {
+        public StorageAvailability(ByteSize totalSpace, TimeSpan maxDuration, TestToken minPriceForTotalSpace, TestToken maxCollateral)
+        {
+            TotalSpace = totalSpace;
+            MaxDuration = maxDuration;
+            MinPriceForTotalSpace = minPriceForTotalSpace;
+            MaxCollateral = maxCollateral;
+        }
+
+        public string Id { get; set; } = string.Empty;
+        public ByteSize TotalSpace { get; }
+        public TimeSpan MaxDuration { get; }
+        public TestToken MinPriceForTotalSpace { get; }
+        public TestToken MaxCollateral { get; } 
+
+        public void Log(ILog log)
+        {
+            log.Log($"Making storage available... (" +
+                $"totalSize: {TotalSpace}, " +
+                $"maxDuration: {Time.FormatDuration(MaxDuration)}, " + 
+                $"minPriceForTotalSpace: {MinPriceForTotalSpace}, " +
+                $"maxCollateral: {MaxCollateral})");
+        }
+    }
+}

ProjectPlugins/CodexPlugin/TransferSpeeds.cs

@@ -0,0 +1,67 @@
+using Utils;
+
+namespace CodexPlugin
+{
+    public interface ITransferSpeeds
+    {
+        BytesPerSecond? GetUploadSpeed();
+        BytesPerSecond? GetDownloadSpeed();
+    }
+
+    public class TransferSpeeds : ITransferSpeeds
+    {
+        private readonly List<BytesPerSecond> uploads = new List<BytesPerSecond>();
+        private readonly List<BytesPerSecond> downloads = new List<BytesPerSecond>();
+
+        public void AddUploadSample(ByteSize bytes, TimeSpan duration)
+        {
+            uploads.Add(Convert(bytes, duration));
+        }
+
+        public void AddDownloadSample(ByteSize bytes, TimeSpan duration)
+        {
+            downloads.Add(Convert(bytes, duration));
+        }
+
+        public BytesPerSecond? GetUploadSpeed()
+        {
+            if (!uploads.Any()) return null;
+            return uploads.Average();
+        }
+
+        public BytesPerSecond? GetDownloadSpeed()
+        {
+            if (!downloads.Any()) return null;
+            return downloads.Average();
+        }
+
+        private static BytesPerSecond Convert(ByteSize size, TimeSpan duration)
+        {
+            double bytes = size.SizeInBytes;
+            double seconds = duration.TotalSeconds;
+
+            return new BytesPerSecond(System.Convert.ToInt64(Math.Round(bytes / seconds)));
+        }
+    }
+
+    public static class ListExtensions
+    {
+        public static BytesPerSecond Average(this List<BytesPerSecond> list)
+        {
+            double sum = list.Sum(i => i.SizeInBytes);
+            double num = list.Count;
+
+            return new BytesPerSecond(Convert.ToInt64(Math.Round(sum / num)));
+        }
+
+        public static BytesPerSecond? OptionalAverage(this List<BytesPerSecond?>? list)
+        {
+            if (list == null || !list.Any() || !list.Any(i => i != null)) return null;
+            var values = list.Where(i => i != null).Cast<BytesPerSecond>().ToArray();
+            double sum = values.Sum(i => i.SizeInBytes);
+            double num = values.Length;
+
+            return new BytesPerSecond(Convert.ToInt64(Math.Round(sum / num)));
+        }
+    }
+}

ProjectPlugins/CodexPlugin/openapi.yaml

@@ -0,0 +1,720 @@
+openapi: 3.0.3
+
+info:
+  version: 0.0.1
+  title: Codex API
+  description: "List of endpoints and interfaces available to Codex API users"
+
+security:
+  - { }
+
+components:
+  schemas:
+    MultiAddress:
+      type: string
+      description: Address of node as specified by the multi-address specification https://multiformats.io/multiaddr/
+      example: /ip4/127.0.0.1/tcp/8080
+
+    PeerId:
+      type: string
+      description: Peer Identity reference as specified at https://docs.libp2p.io/concepts/fundamentals/peers/
+      example: QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N
+
+    Id:
+      type: string
+      description: 32bits identifier encoded in hex-decimal string.
+      example: 0x...
+
+    BigInt:
+      type: string
+      description: Integer represented as decimal string
+
+    Cid:
+      type: string
+      description: Content Identifier as specified at https://github.com/multiformats/cid
+      example: QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N
+
+    SlotId:
+      type: string
+      description: Keccak hash of the abi encoded tuple (RequestId, slot index)
+      example: 268a781e0db3f7cf36b18e5f4fdb7f586ec9edd08e5500b17c0e518a769f114a
+
+    LogLevel:
+      type: string
+      description: "One of the log levels: TRACE, DEBUG, INFO, NOTICE, WARN, ERROR or FATAL"
+      example: DEBUG
+
+    EthereumAddress:
+      type: string
+      description: Address of Ethereum address
+
+    Reward:
+      type: string
+      description: The maximum amount of tokens paid per second per slot to hosts the client is willing to pay
+
+    Duration:
+      type: string
+      description: The duration of the request in seconds as decimal string
+
+    ProofProbability:
+      type: string
+      description: How often storage proofs are required as decimal string
+
+    Expiry:
+      type: string
+      description: A timestamp as seconds since unix epoch at which this request expires if the Request does not find requested amount of nodes to host the data.
+      default: 10 minutes
+
+    ErasureParameters:
+      type: object
+      properties:
+        totalChunks:
+          type: number
+
+    PoRParameters:
+      description: Parameters for Proof of Retrievability
+      type: object
+      properties:
+        u:
+          type: string
+        publicKey:
+          type: string
+        name:
+          type: string
+
+    Content:
+      type: object
+      description: Parameters specifying the content
+      properties:
+        cid:
+          $ref: "#/components/schemas/Cid"
+        erasure:
+          $ref: "#/components/schemas/ErasureParameters"
+        por:
+          $ref: "#/components/schemas/PoRParameters"
+
+    DebugInfo:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/PeerId"
+        addrs:
+          type: array
+          items:
+            $ref: "#/components/schemas/MultiAddress"
+        repo:
+          type: string
+          description: Path of the data repository where all nodes data are stored
+        spr:
+          type: string
+          description: Signed Peer Record to advertise DHT connection information
+
+    SalesAvailability:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/Id"
+        totalSize:
+          type: string
+          description: Total size of availability's storage in bytes as decimal string
+        duration:
+          $ref: "#/components/schemas/Duration"
+        minPrice:
+          type: string
+          description: Minimum price to be paid (in amount of tokens) as decimal string
+        maxCollateral:
+          type: string
+          description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string
+
+    SalesAvailabilityREAD:
+      allOf:
+        - $ref: "#/components/schemas/SalesAvailability"
+        - type: object
+          properties:
+            freeSize:
+              type: string
+              description: Unused size of availability's storage in bytes as decimal string
+
+    SalesAvailabilityCREATE:
+      allOf:
+        - $ref: "#/components/schemas/SalesAvailability"
+        - required:
+            - totalSize
+            - minPrice
+            - maxCollateral
+            - duration
+
+    Slot:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/SlotId"
+        request:
+          $ref: "#/components/schemas/StorageRequest"
+        slotIndex:
+          type: string
+          description: Slot Index as hexadecimal string
+
+    Reservation:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/Id"
+        availabilityId:
+          $ref: "#/components/schemas/Id"
+        size:
+          $ref: "#/components/schemas/BigInt"
+        requestId:
+          $ref: "#/components/schemas/Id"
+        slotIndex:
+          type: string
+          description: Slot Index as hexadecimal string
+
+    StorageRequestCreation:
+      type: object
+      required:
+        - reward
+        - duration
+        - proofProbability
+        - collateral
+        - expiry
+      properties:
+        duration:
+          $ref: "#/components/schemas/Duration"
+        reward:
+          $ref: "#/components/schemas/Reward"
+        proofProbability:
+          $ref: "#/components/schemas/ProofProbability"
+        nodes:
+          type: number
+          description: Minimal number of nodes the content should be stored on
+          default: 1
+        tolerance:
+          type: number
+          description: Additional number of nodes on top of the `nodes` property that can be lost before pronouncing the content lost
+          default: 0
+        collateral:
+          type: string
+          description: Number as decimal string that represents how much collateral is asked from hosts that wants to fill a slots
+        expiry:
+          type: string
+          description: Number as decimal string that represents expiry time of the request (in unix timestamp)
+
+    StorageAsk:
+      type: object
+      required:
+        - reward
+      properties:
+        slots:
+          type: number
+          description: Number of slots (eq. hosts) that the Request want to have the content spread over
+        slotSize:
+          type: string
+          description: Amount of storage per slot (in bytes) as decimal string
+        duration:
+          $ref: "#/components/schemas/Duration"
+        proofProbability:
+          $ref: "#/components/schemas/ProofProbability"
+        reward:
+          $ref: "#/components/schemas/Reward"
+        maxSlotLoss:
+          type: number
+          description: Max slots that can be lost without data considered to be lost
+
+    StorageRequest:
+      type: object
+      properties:
+        id:
+          type: string
+          description: Request ID
+        client:
+          $ref: "#/components/schemas/EthereumAddress"
+        ask:
+          $ref: "#/components/schemas/StorageAsk"
+        content:
+          $ref: "#/components/schemas/Content"
+        expiry:
+          $ref: "#/components/schemas/Expiry"
+        nonce:
+          type: string
+          description: Random data
+
+    Purchase:
+      type: object
+      properties:
+        state:
+          type: string
+          description: Description of the Request's state
+        error:
+          type: string
+          description: If Request failed, then here is presented the error message
+        request:
+          $ref: "#/components/schemas/StorageRequest"
+
+    DataList:
+      type: object
+      properties:
+        content:
+          type: array
+          items:
+            $ref: "#/components/schemas/DataItem"
+
+    DataItem:
+      type: object
+      properties:
+        cid:
+          $ref: "#/components/schemas/Cid"
+        manifest:
+          $ref: "#/components/schemas/ManifestItem"
+
+    ManifestItem:
+      type: object
+      properties:
+        rootHash:
+          $ref: "#/components/schemas/Cid"
+          description: "Root hash of the content"
+        originalBytes:
+          type: number
+          description: "Length of original content in bytes"
+        blockSize:
+          type: number
+          description: "Size of blocks"
+        protected:
+          type: boolean
+          description: "Indicates if content is protected by erasure-coding"
+
+    Space:
+      type: object
+      properties:
+        totalBlocks:
+          type: number
+          description: "Number of blocks stored by the node"
+        quotaMaxBytes:
+          type: number
+          description: "Maximum storage space used by the node"
+        quotaUsedBytes:
+          type: number
+          description: "Amount of storage space currently in use"
+        quotaReservedBytes:
+          type: number
+          description: "Amount of storage space reserved"
+
+servers:
+  - url: "http://localhost:8080/api/codex/v1"
+
+tags:
+  - name: Marketplace
+    description: Marketplace information and operations
+  - name: Data
+    description: Data operations
+  - name: Node
+    description: Node management
+  - name: Debug
+    description: Debugging configuration
+
+paths:
+  "/connect/{peerId}":
+    get:
+      summary: "Connect to a peer"
+      description: |
+        If `addrs` param is supplied, it will be used to dial the peer, otherwise the `peerId` is used
+        to invoke peer discovery, if it succeeds the returned addresses will be used to dial.
+      tags: [ Node ]
+      operationId: connectPeer
+      parameters:
+        - in: path
+          name: peerId
+          required: true
+          schema:
+              $ref: "#/components/schemas/PeerId"
+          description: Peer that should be dialed.
+        - in: query
+          name: addrs
+          schema:
+            type: array
+            nullable: true
+            items:
+              $ref: "#/components/schemas/MultiAddress"
+          description: |
+            If supplied, it will be used to dial the peer.
+            The address has to target the listening address of the peer,
+            which is specified with the `--listen-addrs` CLI flag.
+
+      responses:
+        "200":
+          description: Successfully connected to peer
+        "400":
+          description: Peer either not found or was not possible to dial
+
+  "/data":
+    get:
+      summary: "Lists manifest CIDs stored locally in node."
+      tags: [ Data ]
+      operationId: listData
+      responses:
+        "200":
+          description: Retrieved list of content CIDs
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DataList"
+
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Content specified by the CID is not found
+        "500":
+          description: Well it was bad-bad
+    post:
+      summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID."
+      tags: [ Data ]
+      operationId: upload
+      requestBody:
+        content:
+          application/octet-stream:
+            schema:
+              type: string
+              format: binary
+      responses:
+        "200":
+          description: CID of uploaded file
+          content:
+            text/plain:
+              schema:
+                type: string
+        "500":
+          description: Well it was bad-bad and the upload did not work out
+
+  "/data/{cid}":
+    get:
+      summary: "Download a file from the local node in a streaming manner. If the file is not available locally, a 404 is returned."
+      tags: [ Data ]
+      operationId: downloadLocal
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: File to be downloaded.
+
+      responses:
+        "200":
+          description: Retrieved content specified by CID
+          content:
+            application/octet-stream:
+              schema:
+                type: string
+                format: binary
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Content specified by the CID is unavailable locally
+        "500":
+          description: Well it was bad-bad
+
+  "/data/{cid}/network":
+    get:
+      summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
+      tags: [ Data ]
+      operationId: downloadNetwork
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+              $ref: "#/components/schemas/Cid"
+          description: "File to be downloaded."
+      responses:
+        "200":
+          description: Retrieved content specified by CID
+          content:
+            application/octet-stream:
+              schema:
+                type: string
+                format: binary
+        "400":
+          description: Invalid CID is specified
+        "404":
+          description: Content specified by the CID is not found
+        "500":
+          description: Well it was bad-bad
+
+  "/space":
+    get:
+      summary: "Gets a summary of the storage space allocation of the node."
+      tags: [ Data ]
+      operationId: space
+      responses:
+        "200":
+          description: "Summary of storage allocation"
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Space"
+
+        "500":
+          description: "It's not working as planned"
+
+  "/sales/slots":
+    get:
+      summary: "Returns active slots"
+      tags: [ Marketplace ]
+      operationId: getActiveSlots
+      responses:
+        "200":
+          description: Retrieved active slots
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/Slot"
+
+        "503":
+          description: Sales are unavailable
+
+  "/sales/slots/{slotId}":
+    get:
+      summary: "Returns active slot with id {slotId} for the host"
+      tags: [ Marketplace ]
+      operationId: getActiveSlotById
+      parameters:
+        - in: path
+          name: slotId
+          required: true
+          schema:
+            $ref: "#/components/schemas/Cid"
+          description: File to be downloaded.
+      responses:
+        "200":
+          description: Retrieved active slot
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Slot"
+
+        "400":
+          description: Invalid or missing SlotId
+
+        "404":
+          description: Host is not in an active sale for the slot
+
+        "503":
+          description: Sales are unavailable
+
+  "/sales/availability":
+    get:
+      summary: "Returns storage that is for sale"
+      tags: [ Marketplace ]
+      operationId: getOfferedStorage
+      responses:
+        "200":
+          description: Retrieved storage availabilities of the node
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/SalesAvailability"
+        "500":
+          description: Error getting unused availabilities
+        "503":
+          description: Sales are unavailable
+
+    post:
+      summary: "Offers storage for sale"
+      operationId: offerStorage
+      tags: [ Marketplace ]
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/SalesAvailabilityCREATE"
+      responses:
+        "201":
+          description: Created storage availability
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/SalesAvailabilityREAD"
+        "400":
+          description: Invalid data input
+        "422":
+          description: Not enough node's storage quota available
+        "500":
+          description: Error reserving availability
+        "503":
+          description: Sales are unavailable
+  "/sales/availability/{id}":
+    patch:
+      summary: "Updates availability"
+      description: |
+        The new parameters will be only considered for new requests.
+        Existing Requests linked to this Availability will continue as is.
+      operationId: updateOfferedStorage
+      tags: [ Marketplace ]
+      parameters:
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: string
+          description: ID of Availability
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/SalesAvailability"
+      responses:
+        "204":
+          description: Availability successfully updated
+        "400":
+          description: Invalid data input
+        "404":
+          description: Availability not found
+        "422":
+          description: Not enough node's storage quota available
+        "500":
+          description: Error reserving availability
+        "503":
+          description: Sales are unavailable
+
+  "/sales/availability/{id}/reservations":
+    patch:
+      summary: "Get availability's reservations"
+      description: Return's list of Reservations for ongoing Storage Requests that the node hosts.
+      operationId: getReservations
+      tags: [ Marketplace ]
+      parameters:
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: string
+          description: ID of Availability
+      responses:
+        "200":
+          description: Retrieved storage availabilities of the node
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: "#/components/schemas/Reservation"
+        "400":
+          description: Invalid Availability ID
+        "404":
+          description: Availability not found
+        "500":
+          description: Error getting reservations
+        "503":
+          description: Sales are unavailable
+
+  "/storage/request/{cid}":
+    post:
+      summary: "Creates a new Request for storage"
+      tags: [ Marketplace ]
+      operationId: createStorageRequest
+      parameters:
+        - in: path
+          name: cid
+          required: true
+          schema:
+            $ref: "#/components/schemas/Cid"
+          description: CID of the uploaded data that should be stored
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/StorageRequestCreation"
+      responses:
+        "200":
+          description: Returns the Request ID as decimal string
+          content:
+            text/plain:
+              schema:
+                type: string
+        "400":
+          description: Invalid or missing Request ID
+        "404":
+          description: Request ID not found
+        "503":
+          description: Purchasing is unavailable
+
+  "/storage/purchases":
+    get:
+      summary: "Returns list of purchase IDs"
+      tags: [ Marketplace ]
+      operationId: getPurchases
+      responses:
+        "200":
+          description: Gets all purchase IDs stored in node
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: string
+        "503":
+          description: Purchasing is unavailable
+
+  "/storage/purchases/{id}":
+    get:
+      summary: "Returns purchase details"
+      tags: [ Marketplace ]
+      operationId: getPurchase
+      parameters:
+        - in: path
+          name: id
+          required: true
+          schema:
+            type: string
+          description: Hexadecimal ID of a Purchase
+      responses:
+        "200":
+          description: Purchase details
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/Purchase"
+        "400":
+          description: Invalid or missing Purchase ID
+        "404":
+          description: Purchase not found
+        "503":
+          description: Purchasing is unavailable
+
+  "/debug/chronicles/loglevel":
+    post:
+      summary: "Set log level at run time"
+      tags: [ Debug ]
+      operationId: setDebugLogLevel
+
+      parameters:
+        - in: query
+          name: level
+          required: true
+          schema:
+            $ref: "#/components/schemas/LogLevel"
+
+      responses:
+        "200":
+          description: Successfully log level set
+        "400":
+          description: Invalid or missing log level
+        "500":
+          description: Well it was bad-bad
+
+  "/debug/info":
+    get:
+      summary: "Gets node information"
+      operationId: getDebugInfo
+      tags: [ Debug ]
+      responses:
+        "200":
+          description: Node's information
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/DebugInfo"

ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj

@@ -0,0 +1,10 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+</Project>

ProjectPlugins/CodexPluginPrebuild/Program.cs

@@ -0,0 +1,61 @@
+using System.Security.Cryptography;
+using System.Text;
+
+public static class Program
+{
+    private const string OpenApiFile = "../CodexPlugin/openapi.yaml";
+    private const string Search = "<INSERT-OPENAPI-YAML-HASH>";
+    private const string TargetFile = "ApiChecker.cs";
+
+    public static void Main(string[] args)
+    {
+        Console.WriteLine("Injecting hash of 'openapi.yaml'...");
+
+        var hash = CreateHash();
+        // This hash is used to verify that the Codex docker image being used is compatible
+        // with the openapi.yaml being used by the Codex plugin.
+        // If the openapi.yaml files don't match, an exception is thrown.
+
+        SearchAndInject(hash);
+
+        // This program runs as the pre-build trigger for "CodexPlugin".
+        // You might be wondering why this work isn't done by a shell script.
+        // This is because this project is being run on many different platforms.
+        // (Mac, Unix, Win, but also desktop/cloud containers.)
+        // In order to not go insane trying to make a shell script that works in all possible cases,
+        // instead we use the one tool that's definitely installed in all platforms and locations
+        // when you're trying to run this plugin.
+
+        Console.WriteLine("Done!");
+    }
+
+    private static string CreateHash()
+    {
+        var file = File.ReadAllText(OpenApiFile);
+        var fileBytes = Encoding.ASCII.GetBytes(file
+            .Replace(Environment.NewLine, ""));
+
+        var sha = SHA256.Create();
+        var hash = sha.ComputeHash(fileBytes);
+        return BitConverter.ToString(hash);
+    }
+
+    private static void SearchAndInject(string hash)
+    {
+        var lines = File.ReadAllLines(TargetFile);
+        Inject(lines, hash);
+        File.WriteAllLines(TargetFile, lines);
+    }
+
+    private static void Inject(string[] lines, string hash)
+    {
+        for (var i = 0; i < lines.Length; i++)
+        {
+            if (lines[i].Contains(Search))
+            {
+                lines[i + 1] = $"        private const string OpenApiYamlHash = \"{hash}\";";
+                return;
+            }
+        }
+    }
+}

ProjectPlugins/GethPlugin/EthAccount.cs

@@ -0,0 +1,28 @@
+using Nethereum.Hex.HexConvertors.Extensions;
+using Nethereum.Web3.Accounts;
+
+namespace GethPlugin
+{
+    [Serializable]
+    public class EthAccount
+    {
+        public EthAccount(EthAddress ethAddress, string privateKey)
+        {
+            EthAddress = ethAddress;
+            PrivateKey = privateKey;
+        }
+
+        public EthAddress EthAddress { get; }
+        public string PrivateKey { get; }
+
+        public static EthAccount GenerateNew()
+        {
+            var ecKey = Nethereum.Signer.EthECKey.GenerateKey();
+            var privateKey = ecKey.GetPrivateKeyAsBytes().ToHex();
+            var account = new Account(privateKey);
+            var ethAddress = new EthAddress(account.Address);
+
+            return new EthAccount(ethAddress, account.PrivateKey);
+        }
+    }
+}

ProjectPlugins/GethPlugin/EthAddress.cs

@@ -5,15 +5,27 @@
         EthAddress EthAddress { get; }
     }
 
+    [Serializable]
     public class EthAddress
     {
         public EthAddress(string address)
         {
-            Address = address;
+            Address = address.ToLowerInvariant();
         }
 
         public string Address { get; }
 
+        public override bool Equals(object? obj)
+        {
+            return obj is EthAddress address &&
+                   Address == address.Address;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Address);
+        }
+
         public override string ToString()
         {
             return Address;

ProjectPlugins/GethPlugin/GethContainerRecipe.cs

@@ -24,13 +24,19 @@ namespace GethPlugin
 
             var args = CreateArgs(config);
 
+            SetSchedulingAffinity(notIn: "false");
+            SetSystemCriticalPriority();
+
             AddEnvVar("GETH_ARGS", args);
         }
 
         private string CreateArgs(GethStartupConfig config)
         {
-            if (config.IsMiner) AddEnvVar("ENABLE_MINER", "1");
-            UnlockAccounts(0, 1);
+            if (config.IsMiner)
+            {
+                AddEnvVar("ENABLE_MINER", "1");
+                UnlockAccounts(0, 1);
+            }
 
             var httpPort = CreateApiPort(config, tag: HttpPortTag);
             var discovery = CreateDiscoveryPort(config);
@@ -38,16 +44,20 @@ namespace GethPlugin
             var authRpc = CreateP2pPort(config, tag: AuthRpcPortTag);
             var wsPort = CreateP2pPort(config, tag: WsPortTag);
 
-            var args = $"--http.addr 0.0.0.0 --http.port {httpPort.Number} --port {listen.Number} --discovery.port {discovery.Number} {GetTestNetArgs(config)} {defaultArgs}";
+            var args = $"--http.addr 0.0.0.0 --http.port {httpPort.Number} --port {listen.Number} --discovery.port {discovery.Number} {defaultArgs}";
 
             if (config.BootstrapNode != null)
             {
                 var bootPubKey = config.BootstrapNode.PublicKey;
                 var bootIp = config.BootstrapNode.IpAddress;
                 var bootPort = config.BootstrapNode.Port;
-                var bootstrapArg = $" --bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}";
+                var bootstrapArg = $" --bootnodes enode://{bootPubKey}@{bootIp}:{bootPort}";
                 args += bootstrapArg;
             }
+            if (config.IsPublicTestNet != null)
+            {
+                AddEnvVar("NAT_PUBLIC_IP_AUTO", PublicIpService.Address);
+            }
 
             return args + $" --authrpc.port {authRpc.Number} --ws --ws.addr 0.0.0.0 --ws.port {wsPort.Number}";
         }
@@ -55,20 +65,13 @@ namespace GethPlugin
         private void UnlockAccounts(int startIndex, int numberOfAccounts)
         {
             if (startIndex < 0) throw new ArgumentException();
-            if (numberOfAccounts < 1) throw new ArgumentException();
+            if (numberOfAccounts < 0) throw new ArgumentException();
             if (startIndex + numberOfAccounts > 1000) throw new ArgumentException("Out of accounts!");
 
             AddEnvVar("UNLOCK_START_INDEX", startIndex.ToString());
             AddEnvVar("UNLOCK_NUMBER", numberOfAccounts.ToString());
         }
 
-        private string GetTestNetArgs(GethStartupConfig config)
-        {
-            if (config.IsPublicTestNet == null) return string.Empty;
-
-            return $"--nat=extip:{config.IsPublicTestNet.PublicIp}";
-        }
-
         private Port CreateDiscoveryPort(GethStartupConfig config)
         {
             if (config.IsPublicTestNet == null) return AddInternalPort(DiscoveryPortTag);

ProjectPlugins/GethPlugin/GethNode.cs

@@ -1,8 +1,11 @@
 using Core;
 using KubernetesWorkflow.Types;
 using Logging;
+using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
+using Nethereum.RPC.Eth.DTOs;
 using NethereumWorkflow;
+using Utils;
 
 namespace GethPlugin
 {
@@ -13,19 +16,23 @@ namespace GethPlugin
         Ether GetEthBalance();
         Ether GetEthBalance(IHasEthAddress address);
         Ether GetEthBalance(EthAddress address);
-        void SendEth(IHasEthAddress account, Ether eth);
-        void SendEth(EthAddress account, Ether eth);
+        string SendEth(IHasEthAddress account, Ether eth);
+        string SendEth(EthAddress account, Ether eth);
         TResult Call<TFunction, TResult>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new();
-        void SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new();
+        string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new();
+        Transaction GetTransaction(string transactionHash);
         decimal? GetSyncedBlockNumber();
         bool IsContractAvailable(string abi, string contractAddress);
+        GethBootstrapNode GetBootstrapRecord();
+        List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new();
+        List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new();
     }
 
-    public class GethNode : IGethNode
+    public class DeploymentGethNode : BaseGethNode, IGethNode
     {
         private readonly ILog log;
 
-        public GethNode(ILog log, GethDeployment startResult)
+        public DeploymentGethNode(ILog log, GethDeployment startResult)
         {
             this.log = log;
             StartResult = startResult;
@@ -34,6 +41,59 @@ namespace GethPlugin
         public GethDeployment StartResult { get; }
         public RunningContainer Container => StartResult.Container;
 
+        public GethBootstrapNode GetBootstrapRecord()
+        {
+            var address = StartResult.Container.GetInternalAddress(GethContainerRecipe.ListenPortTag);
+
+            return new GethBootstrapNode(
+                publicKey: StartResult.PubKey,
+                ipAddress: address.Host.Replace("http://", ""),
+                port: address.Port
+            );
+        }
+
+        protected override NethereumInteraction StartInteraction()
+        {
+            var address = StartResult.Container.GetAddress(log, GethContainerRecipe.HttpPortTag);
+            var account = StartResult.Account;
+
+            var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);
+            return creator.CreateWorkflow();
+        }
+    }
+
+    public class CustomGethNode : BaseGethNode, IGethNode
+    {
+        private readonly ILog log;
+        private readonly string gethHost;
+        private readonly int gethPort;
+        private readonly string privateKey;
+
+        public GethDeployment StartResult => throw new NotImplementedException();
+        public RunningContainer Container => throw new NotImplementedException();
+
+        public CustomGethNode(ILog log, string gethHost, int gethPort, string privateKey)
+        {
+            this.log = log;
+            this.gethHost = gethHost;
+            this.gethPort = gethPort;
+            this.privateKey = privateKey;
+        }
+
+        public GethBootstrapNode GetBootstrapRecord()
+        {
+            throw new NotImplementedException();
+        }
+
+        protected override NethereumInteraction StartInteraction()
+        {
+            var creator = new NethereumInteractionCreator(log, gethHost, gethPort, privateKey);
+            return creator.CreateWorkflow();
+        }
+    }
+
+    public abstract class BaseGethNode
+    {
         public Ether GetEthBalance()
         {
             return StartInteraction().GetEthBalance().Eth();
@@ -49,14 +109,14 @@ namespace GethPlugin
             return StartInteraction().GetEthBalance(address.Address).Eth();
         }
 
-        public void SendEth(IHasEthAddress owner, Ether eth)
+        public string SendEth(IHasEthAddress owner, Ether eth)
         {
-            SendEth(owner.EthAddress, eth);
+            return SendEth(owner.EthAddress, eth);
         }
 
-        public void SendEth(EthAddress account, Ether eth)
+        public string SendEth(EthAddress account, Ether eth)
         {
-            StartInteraction().SendEth(account.Address, eth.Eth);
+            return StartInteraction().SendEth(account.Address, eth.Eth);
         }
 
         public TResult Call<TFunction, TResult>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
@@ -64,18 +124,14 @@ namespace GethPlugin
             return StartInteraction().Call<TFunction, TResult>(contractAddress, function);
         }
 
-        public void SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
+        public string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
         {
-            StartInteraction().SendTransaction(contractAddress, function);
+            return StartInteraction().SendTransaction(contractAddress, function);
         }
 
-        private NethereumInteraction StartInteraction()
+        public Transaction GetTransaction(string transactionHash)
         {
-            var address = StartResult.Container.GetAddress(log, GethContainerRecipe.HttpPortTag);
-            var account = StartResult.Account;
-
-            var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);
-            return creator.CreateWorkflow();
+            return StartInteraction().GetTransaction(transactionHash);
         }
 
         public decimal? GetSyncedBlockNumber()
@@ -87,5 +143,17 @@ namespace GethPlugin
         {
             return StartInteraction().IsContractAvailable(abi, contractAddress);
         }
+
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new()
+        {
+            return StartInteraction().GetEvents<TEvent>(address, fromBlockNumber, toBlockNumber);
+        }
+
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new()
+        {
+            return StartInteraction().GetEvents<TEvent>(address, timeRange);
+        }
+
+        protected abstract NethereumInteraction StartInteraction();
     }
 }

ProjectPlugins/GethPlugin/GethStarter.cs

@@ -44,7 +44,7 @@ namespace GethPlugin
         public IGethNode WrapGethContainer(GethDeployment startResult)
         {
             startResult = SerializeGate.Gate(startResult);
-            return new GethNode(tools.GetLog(), startResult);
+            return new DeploymentGethNode(tools.GetLog(), startResult);
         }
 
         private void Log(string msg)

ProjectPlugins/GethPlugin/GethStartupConfig.cs

@@ -3,6 +3,7 @@
     public interface IGethSetup
     {
         IGethSetup IsMiner();
+        IGethSetup WithBootstrapNode(IGethNode node);
         IGethSetup WithBootstrapNode(GethBootstrapNode node);
         IGethSetup WithName(string name);
         IGethSetup AsPublicTestNet(GethTestNetConfig gethTestNetConfig);
@@ -15,6 +16,11 @@
         public string? NameOverride { get; private set; }
         public GethTestNetConfig? IsPublicTestNet { get; private set; }
 
+        public IGethSetup WithBootstrapNode(IGethNode node)
+        {
+            return WithBootstrapNode(node.GetBootstrapRecord());
+        }
+
         public IGethSetup WithBootstrapNode(GethBootstrapNode node)
         {
             BootstrapNode = node;
@@ -42,14 +48,12 @@
 
     public class GethTestNetConfig
     {
-        public GethTestNetConfig(string publicIp, int discoveryPort, int listenPort)
+        public GethTestNetConfig(int discoveryPort, int listenPort)
         {
-            PublicIp = publicIp;
             DiscoveryPort = discoveryPort;
             ListenPort = listenPort;
         }
 
-        public string PublicIp { get; }
         public int DiscoveryPort { get; }
         public int ListenPort { get; }
     }

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -7,14 +7,16 @@ namespace MetricsPlugin
 {
     public class MetricsQuery
     {
-        private readonly IHttp http;
+        private readonly IEndpoint endpoint;
         private readonly ILog log;
 
         public MetricsQuery(IPluginTools tools, RunningContainer runningContainer)
         {
             RunningContainer = runningContainer;
             log = tools.GetLog();
-            http = tools.CreateHttp(RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag), "api/v1");
+            endpoint = tools
+                .CreateHttp()
+                .CreateEndpoint(RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag), "/api/v1/");
         }
 
         public RunningContainer RunningContainer { get; }
@@ -51,7 +53,7 @@ namespace MetricsPlugin
 
         public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
         {
-            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
+            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
             if (response.status != "success") return null;
             var result = MapResponseToMetrics(response);
             Log(target, result);
@@ -60,14 +62,14 @@ namespace MetricsPlugin
 
         private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
         {
-            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})");
+            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})");
             if (response.status != "success") return null;
             return response;
         }
 
         private PrometheusQueryResponse? GetAll(string metricName)
         {
-            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}");
+            var response = endpoint.HttpGetJson<PrometheusQueryResponse>($"query?query={metricName}{GetQueryTimeRange()}");
             if (response.status != "success") return null;
             return response;
         }

ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs

@@ -14,6 +14,8 @@ namespace MetricsPlugin
         {
             var config = startupConfig.Get<PrometheusStartupConfig>();
 
+            SetSchedulingAffinity(notIn: "false");
+
             AddExposedPortAndVar("PROM_PORT", PortTag);
             AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
         }

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -18,6 +18,8 @@ namespace MetricsPlugin
 
         public RunningContainers CollectMetricsFor(IMetricsScrapeTarget[] targets)
         {
+            if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
+
             Log($"Starting metrics server for {targets.Length} targets...");
             var startupConfig = new StartupConfig();
             startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));

README.md

@@ -1,18 +1,25 @@
-# Distributed System Tests for Nim-Codex
+# Distributed System Tests
 
-Using a common dotnet unit-test framework and a few other libraries, this project allows you to write tests that use multiple Codex node instances in various configurations to test the distributed system in a controlled, reproducible environment.
+This project allows you to write tools and tests that control and interact with container-based applications to form a distributed system in a controlled, reproducible environment.
 
-Nim-Codex: https://github.com/codex-storage/nim-codex  
 Dotnet: v7.0  
 Kubernetes: v1.25.4  
 Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp  
 Nethereum: v4.14.0
 
+Currently, this project is mainly used for distributed testing of [Nim-Codex](https://github.com/codex-storage/nim-codex). However, its plugin-structure allows for other projects to be on-boarded (relatively) easily. (See 'contribute a plugin`.)
+
+## Tests/DistTestCore
+Library with generic distributed-testing functionality. Uses NUnit3. Reference this project to build unit-test style scenarios: setup, run test, teardown. The DistTestCore responds to the following env-vars:
+- `LOGPATH` = Path where log files will be written.
+- `DATAFILEPATH` = Path where (temporary) data files will be stored.
+- `ALWAYS_LOGS` = When set, DistTestCore will always download all container logs at the end of a test run. By default, logs are only downloaded on test failure.
+
 ## Tests/CodexTests and Tests/CodexLongTests
-These are test assemblies that use NUnit3 to perform tests against transient Codex nodes.
+These are test assemblies that use DistTestCore to perform tests against transient Codex nodes.
 Read more [HERE](/Tests/CodexTests/README.md)
 
-## Tests/ContinousTests
+## Tests/ContinuousTests
 A console application that runs tests in an endless loop against a persistent deployment of Codex nodes.
 Read more [HERE](/Tests/CodexContinuousTests/README.md)
 

Tests/CodexContinuousTests/Configuration.cs

@@ -24,8 +24,8 @@ namespace ContinuousTests
         [Uniform("stop", "s", "STOPONFAIL", false, "If greater than zero, runner will stop after this many test failures and download all cluster container logs. 0 by default.")]
         public int StopOnFailure { get; set; } = 0;
 
-        [Uniform("target-duration", "td", "TARGETDURATION", false, "If greater than zero, runner will run for this many seconds before stopping.")]
-        public int TargetDurationSeconds { get; set; } = 0;
+        [Uniform("target-duration", "td", "TARGETDURATION", false, "If set, runner will run for this length of time before stopping. Supports seconds, or '1d2h3m4s' format.")]
+        public string TargetDurationSeconds { get; set; } = string.Empty;
 
         [Uniform("filter", "f", "FILTER", false, "If set, runs only tests whose names contain any of the filter strings. Comma-separated. Case sensitive.")]
         public string Filter { get; set; } = string.Empty;

Tests/CodexContinuousTests/ContinuousTestRunner.cs

@@ -1,7 +1,9 @@
-using DistTestCore.Logs;
+using DistTestCore;
+using DistTestCore.Logs;
 using Logging;
 using Newtonsoft.Json;
 using Utils;
+using TaskFactory = Utils.TaskFactory;
 
 namespace ContinuousTests
 {
@@ -25,10 +27,11 @@ namespace ContinuousTests
             var startTime = DateTime.UtcNow;
 
             var overviewLog = new LogSplitter(
-                new FixtureLog(logConfig, startTime, "Overview"),
+                new FixtureLog(logConfig, startTime, config.CodexDeployment.Id, "Overview"),
                 new ConsoleLog()
             );
-            var statusLog = new StatusLog(logConfig, startTime, "continuous-tests", "ContinuousTestRun");
+            var statusLog = new StatusLog(logConfig, startTime, "continuous-tests", config.CodexDeployment.Id,
+                "ContinuousTestRun");
 
             overviewLog.Log("Initializing...");
 
@@ -43,6 +46,7 @@ namespace ContinuousTests
             var taskFactory = new TaskFactory();
             overviewLog.Log("Startup checks passed. Configuration:");
             overviewLog.Log(JsonConvert.SerializeObject(config, Formatting.Indented));
+            overviewLog.Log("Test framework revision: " + GitInfo.GetStatus());
             overviewLog.Log("Continuous tests starting...");
             overviewLog.Log("");
             var allTests = testFactory.CreateTests();
@@ -57,7 +61,7 @@ namespace ContinuousTests
             }
             else
             {
-                var testLoops = filteredTests.Select(t => new TestLoop(entryPointFactory, taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
+                var testLoops = filteredTests.Select(t => new TestLoop(entryPointFactory, taskFactory, config, overviewLog, statusLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
 
                 foreach (var testLoop in testLoops)
                 {
@@ -96,19 +100,20 @@ namespace ContinuousTests
 
         private void WaitUntilFinished(LogSplitter overviewLog, StatusLog statusLog, DateTime startTime, TestLoop[] testLoops)
         {
-            var testDuration = Time.FormatDuration(DateTime.UtcNow - startTime);
+            var testDuration = (DateTime.UtcNow - startTime).TotalSeconds.ToString();
             var testData = FormatTestRuns(testLoops);
             overviewLog.Log("Total duration: " + testDuration);
 
-            if (config.TargetDurationSeconds > 0)
+            if (!string.IsNullOrEmpty(config.TargetDurationSeconds))
             {
-                var targetDuration = TimeSpan.FromSeconds(config.TargetDurationSeconds);
+                var targetDuration = Time.ParseTimespan(config.TargetDurationSeconds);
                 var wasCancelled = cancelToken.WaitHandle.WaitOne(targetDuration);
                 if (!wasCancelled)
                 {
                     Cancellation.Cts.Cancel();
                     overviewLog.Log($"Congratulations! The targer duration has been reached! ({Time.FormatDuration(targetDuration)})");
                     statusLog.ConcludeTest("Passed", testDuration, testData);
+                    Environment.ExitCode = 0;
                     return;
                 }
             }
@@ -117,6 +122,7 @@ namespace ContinuousTests
                 cancelToken.WaitHandle.WaitOne();
             }
             statusLog.ConcludeTest("Failed", testDuration, testData);
+            Environment.ExitCode = 1;
         }
 
         private Dictionary<string, string> FormatTestRuns(TestLoop[] testLoops)
@@ -124,7 +130,8 @@ namespace ContinuousTests
             var result = new Dictionary<string, string>();
             foreach (var testLoop in testLoops)
             {
-                result.Add($"ctest-{testLoop.Name}", $"passes: {testLoop.NumberOfPasses} - failures: {testLoop.NumberOfFailures}");
+                result.Add("testname", testLoop.Name);
+                result.Add($"summary", $"passes: {testLoop.NumberOfPasses} - failures: {testLoop.NumberOfFailures}");
             }
             return result;
         }

Tests/CodexContinuousTests/Program.cs

@@ -4,7 +4,7 @@ public class Program
 {
     public static void Main(string[] args)
     {
-        Console.WriteLine("Codex Continous-Test-Runner.");
+        Console.WriteLine("Codex Continuous-Test-Runner.");
 
         var runner = new ContinuousTestRunner(args, Cancellation.Cts.Token);
 
@@ -15,8 +15,15 @@ public class Program
 
             Cancellation.Cts.Cancel();
         };
-        
-        runner.Run();
+
+        try
+        {
+            runner.Run();
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine(ex.ToString());
+        }
         Console.WriteLine("Done.");
     }
 }

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -6,6 +6,7 @@ using CodexPlugin;
 using DistTestCore.Logs;
 using Core;
 using KubernetesWorkflow.Types;
+using TaskFactory = Utils.TaskFactory;
 
 namespace ContinuousTests
 {
@@ -16,6 +17,7 @@ namespace ContinuousTests
         private readonly TaskFactory taskFactory;
         private readonly Configuration config;
         private readonly ILog overviewLog;
+        private readonly StatusLog statusLog;
         private readonly TestHandle handle;
         private readonly CancellationToken cancelToken;
         private readonly ICodexNode[] nodes;
@@ -23,16 +25,20 @@ namespace ContinuousTests
         private readonly string testName;
         private static int failureCount = 0;
 
-        public SingleTestRun(EntryPointFactory entryPointFactory, TaskFactory taskFactory, Configuration config, ILog overviewLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken)
+        public SingleTestRun(EntryPointFactory entryPointFactory,
+            TaskFactory taskFactory, Configuration config, ILog overviewLog, StatusLog statusLog, TestHandle handle,
+            StartupChecker startupChecker, CancellationToken cancelToken, string deployId)
         {
             this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
+            this.statusLog = statusLog;
             this.handle = handle;
             this.cancelToken = cancelToken;
             testName = handle.Test.GetType().Name;
-            fixtureLog = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, testName);
-            entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, fixtureLog);
+            fixtureLog = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, deployId, testName);
+            entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath,
+                config.CodexDeployment.Metadata.KubeNamespace, fixtureLog);
             ApplyLogReplacements(fixtureLog, startupChecker);
 
             nodes = CreateRandomNodes();
@@ -57,30 +63,34 @@ namespace ContinuousTests
                     overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex);
                     Environment.Exit(-1);
                 }
-            });
+            }, nameof(SingleTestRun));
         }
 
         private void RunTest(Action<bool> resultHandler)
         {
             var testStart = DateTime.UtcNow;
-            
+            TimeSpan duration = TimeSpan.Zero;
+
             try
             {
                 RunTestMoments();
+                duration = DateTime.UtcNow - testStart;
 
-                var duration = DateTime.UtcNow - testStart;
                 OverviewLog($" > Test passed. ({Time.FormatDuration(duration)})");
+                UpdateStatusLogPassed(testStart, duration);
 
                 if (!config.KeepPassedTestLogs)
                 {
                     fixtureLog.Delete();
                 }
+
                 resultHandler(true);
             }
             catch (Exception ex)
             {
                 fixtureLog.Error("Test run failed with exception: " + ex);
                 fixtureLog.MarkAsFailed();
+                UpdateStatusLogFailed(testStart, duration, ex.ToString());
 
                 DownloadContainerLogs(testStart);
 
@@ -108,6 +118,7 @@ namespace ContinuousTests
             {
                 effectiveStart = config.CodexDeployment.Metadata.StartUtc.Subtract(TimeSpan.FromSeconds(30));
             }
+
             var effectiveEnd = DateTime.UtcNow;
             var elasticSearchLogDownloader = new ElasticSearchLogDownloader(entryPoint.Tools, fixtureLog);
 
@@ -116,14 +127,17 @@ namespace ContinuousTests
                 var container = node.Container;
                 var deploymentName = container.RunningContainers.StartResult.Deployment.Name;
                 var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
-                var openingLine = $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().id}";
-                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart, effectiveEnd, openingLine);
+                var openingLine =
+                    $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().id}";
+                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart,
+                    effectiveEnd, openingLine);
             }
         }
 
         private void ApplyLogReplacements(FixtureLog fixtureLog, StartupChecker startupChecker)
         {
-            foreach (var replacement in startupChecker.LogReplacements) fixtureLog.AddStringReplace(replacement.From, replacement.To);
+            foreach (var replacement in startupChecker.LogReplacements)
+                fixtureLog.AddStringReplace(replacement.From, replacement.To);
         }
 
         private void RunTestMoments()
@@ -155,9 +169,11 @@ namespace ContinuousTests
                     {
                         ThrowFailTest();
                     }
+
                     return;
                 }
             }
+
             fixtureLog.Log("Test run has been cancelled.");
         }
 
@@ -170,6 +186,37 @@ namespace ContinuousTests
             throw new Exception(exceptionsMessage);
         }
 
+        private void UpdateStatusLogFailed(DateTime testStart, TimeSpan duration, string error)
+        {
+            statusLog.ConcludeTest("Failed", duration, CreateStatusLogData(testStart, error));
+        }
+
+        private void UpdateStatusLogPassed(DateTime testStart, TimeSpan duration)
+        {
+            statusLog.ConcludeTest("Passed", duration, CreateStatusLogData(testStart, "OK"));
+        }
+
+        private Dictionary<string, string> CreateStatusLogData(DateTime testStart, string message)
+        {
+            var result = entryPoint.GetPluginMetadata();
+            result.Add("teststart", testStart.ToString("o"));
+            result.Add("testname", testName);
+            result.Add("message", message);
+            result.Add("involvedpods", string.Join(",", nodes.Select(n => n.GetName())));
+            result.Add("involvedpodnames", string.Join(",", nodes.Select(n => n.GetPodInfo().Name)));
+
+            var error = message.Split(Environment.NewLine).First();
+            if (error.Contains(":")) error = error.Substring(1 + error.LastIndexOf(":"));
+            result.Add("error", error);
+
+            var upload = nodes.Select(n => n.TransferSpeeds.GetUploadSpeed()).ToList()!.OptionalAverage();
+            var download = nodes.Select(n => n.TransferSpeeds.GetDownloadSpeed()).ToList()!.OptionalAverage();
+            if (upload != null) result.Add("avgupload", upload.ToString());
+            if (download != null) result.Add("avgdownload", download.ToString());
+
+            return result;
+        }
+
         private string GetCombinedExceptionsMessage(Exception[] exceptions)
         {
             return string.Join(Environment.NewLine, exceptions.Select(ex => ex.ToString()));
@@ -186,6 +233,7 @@ namespace ContinuousTests
             {
                 return UnpackException(a.InnerExceptions.First());
             }
+
             if (exception is TargetInvocationException t)
             {
                 return UnpackException(t.InnerException!);
@@ -258,7 +306,8 @@ namespace ContinuousTests
             {
                 result[i] = containers.PickOneRandom();
             }
+
             return result;
         }
     }
-}
+}

Tests/CodexContinuousTests/StartupChecker.cs

@@ -23,7 +23,8 @@ namespace ContinuousTests
 
         public void Check()
         {
-            var log = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, "StartupChecks");
+            var log = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, config.CodexDeployment.Id,
+                "StartupChecks");
             log.Log("Starting continuous test run...");
             IncludeDeploymentConfiguration(log);
             log.Log("Checking configuration...");
@@ -46,13 +47,15 @@ namespace ContinuousTests
                 {
                     var podInfo = workflow.GetPodInfo(container);
                     log.Log($"Codex environment variables for '{container.Name}':");
-                    log.Log($"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
+                    log.Log(
+                        $"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
                         $"Pod name: {podInfo.Name} - Deployment name: {instance.Containers.StartResult.Deployment.Name}");
                     var codexVars = container.Recipe.EnvVars;
                     foreach (var vars in codexVars) log.Log(vars.ToString());
                     log.Log("");
                 }
             }
+
             log.Log($"Deployment metadata: {JsonConvert.SerializeObject(deployment.Metadata)}");
             log.Log("");
         }
@@ -64,6 +67,7 @@ namespace ContinuousTests
             {
                 throw new Exception("Unable to find any tests.");
             }
+
             foreach (var test in tests)
             {
                 cancelToken.ThrowIfCancellationRequested();
@@ -87,7 +91,8 @@ namespace ContinuousTests
 
         private void CheckCodexNodes(BaseLog log, Configuration config)
         {
-            var nodes = entryPoint.CreateInterface().WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray());
+            var nodes = entryPoint.CreateInterface()
+                .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray());
             var pass = true;
             foreach (var n in nodes)
             {
@@ -106,6 +111,7 @@ namespace ContinuousTests
                     pass = false;
                 }
             }
+
             if (!pass)
             {
                 throw new Exception("Not all codex nodes responded.");
@@ -126,6 +132,7 @@ namespace ContinuousTests
             {
                 return false;
             }
+
             return true;
         }
 
@@ -145,7 +152,8 @@ namespace ContinuousTests
                 propertyName: nameof(ContinuousTest.CustomK8sNamespace));
         }
 
-        private void DuplicatesCheck(ContinuousTest[] tests, List<string> errors, Func<ContinuousTest, bool> considerCondition, Func<ContinuousTest, object> getValue, string propertyName)
+        private void DuplicatesCheck(ContinuousTest[] tests, List<string> errors,
+            Func<ContinuousTest, bool> considerCondition, Func<ContinuousTest, object> getValue, string propertyName)
         {
             foreach (var test in tests)
             {
@@ -155,7 +163,8 @@ namespace ContinuousTests
                     if (duplicates.Any())
                     {
                         duplicates.Add(test);
-                        errors.Add($"Tests '{string.Join(",", duplicates.Select(d => d.Name))}' have the same '{propertyName}'. These must be unique.");
+                        errors.Add(
+                            $"Tests '{string.Join(",", duplicates.Select(d => d.Name))}' have the same '{propertyName}'. These must be unique.");
                         return;
                     }
                 }
@@ -170,14 +179,16 @@ namespace ContinuousTests
                 {
                     if (test.RequiredNumberOfNodes < 1)
                     {
-                        errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Test must require > 0 nodes, or -1 to select all nodes.");
+                        errors.Add(
+                            $"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Test must require > 0 nodes, or -1 to select all nodes.");
                     }
                     else if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexInstances.Length)
                     {
-                        errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexInstances.Length}");
+                        errors.Add(
+                            $"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexInstances.Length}");
                     }
                 }
             }
         }
     }
-}
+}

Tests/CodexContinuousTests/TestLoop.cs

@@ -1,4 +1,6 @@
-using Logging;
+using DistTestCore.Logs;
+using Logging;
+using TaskFactory = Utils.TaskFactory;
 
 namespace ContinuousTests
 {
@@ -8,6 +10,7 @@ namespace ContinuousTests
         private readonly TaskFactory taskFactory;
         private readonly Configuration config;
         private readonly ILog overviewLog;
+        private readonly StatusLog statusLog;
         private readonly Type testType;
         private readonly TimeSpan runsEvery;
         private readonly StartupChecker startupChecker;
@@ -15,12 +18,13 @@ namespace ContinuousTests
         private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset);
         private static object testLock = new object();
 
-        public TestLoop(EntryPointFactory entryPointFactory, TaskFactory taskFactory, Configuration config, ILog overviewLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken)
+        public TestLoop(EntryPointFactory entryPointFactory, TaskFactory taskFactory, Configuration config, ILog overviewLog, StatusLog statusLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken)
         {
             this.entryPointFactory = entryPointFactory;
             this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
+            this.statusLog = statusLog;
             this.testType = testType;
             this.runsEvery = runsEvery;
             this.startupChecker = startupChecker;
@@ -66,14 +70,15 @@ namespace ContinuousTests
                     overviewLog.Error("Test infra failure: TestLoop failed with " + ex);
                     Environment.Exit(-1);
                 }
-            });
+            }, nameof(TestLoop));
         }
 
         private void StartTest()
         {
             var test = (ContinuousTest)Activator.CreateInstance(testType)!;
             var handle = new TestHandle(test);
-            var run = new SingleTestRun(entryPointFactory, taskFactory, config, overviewLog, handle, startupChecker, cancelToken);
+            var run = new SingleTestRun(entryPointFactory, taskFactory, config, overviewLog, statusLog, handle,
+                startupChecker, cancelToken, config.CodexDeployment.Id);
 
             runFinishedHandle.Reset();
             run.Run(runFinishedHandle, result =>

Tests/CodexContinuousTests/reports/CodexTestNetReport-August2023.md

@@ -13,7 +13,7 @@ Report for: 08-2023
 (Stopped: The number of tests that can successfully run on the test-net is not high enough to justify the cost of leaving it running.)
 
 ## Deployment Configuration
-Continous Test-net is deployed to the kubernetes cluster with the following configuration:
+Continuous Test-net is deployed to the kubernetes cluster with the following configuration:
 
 5x Codex Nodes:
 - Log-level: Trace

Tests/CodexContinuousTests/reports/CodexTestNetReport-July2023.md

@@ -11,7 +11,7 @@ Report for: 07-2023
 (Faulted: Tests fail with such frequency that the information gathered does not justify the cost of leaving the test-net running.)
 
 ## Deployment Configuration
-Continous Test-net is deployed to the kubernetes cluster with the following configuration:
+Continuous Test-net is deployed to the kubernetes cluster with the following configuration:
 
 5x Codex Nodes:
 - Log-level: Trace

Tests/CodexContinuousTests/reports/CodexTestNetReport-November2023.md

@@ -0,0 +1,41 @@
+# Codex Continuous Test-net Report
+Date: 05-12-2023
+
+Report for: 11-2023
+
+
+## Continuous test-net Status
+Continuous test runs (which can take many hours or days) can easily be started by team member from the github actions UI. Results are collected and displayed in Grafana. For the time being, we're suspending the effort to have a network of Codex nodes "always online" and continuously being tested, until overal reliability improves.
+
+## Deployment Configuration
+Continous Test-net is deployed to the kubernetes cluster with the following configuration:
+
+5x Codex Nodes:
+- Log-level: Trace
+- Storage quota: 20480 MB
+- Storage sell: 1024 MB
+- Min price: 1024
+- Max collateral: 1024
+- Max duration: 3600000 seconds
+- Block-TTL*: 99999999 seconds
+- Block-MI*: 99999999 seconds
+- Block-MN*: 100 blocks
+3 of these 5 nodes have:
+- Validator: true
+
+## Test Overview
+| Changes | Test             | Description                    | Status  | Results              |
+|---------|------------------|--------------------------------|---------|----------------------|
+| todo    | Two-client test  | See report for July 2023.      | Faulted | Test reliably fails. |
+| todo    | Two-client test* | See report for September 2023. | Faulted | Test reliably fails. |
+| todo    | HoldMyBeer test  | See report for August 2023.    | todo    | todo                 |
+| todo    | Peers test       | See report for August 2023.    | todo    | todo                 |
+
+## Resulting changes
+As a result of the testing efforts in 11-2023, these changes were made:
+1. todo
+
+## Action Points
+- Debugging efforts continuou
+- 
+