working runlength implementation and tests

run-length encoding wins
Update marketplace bytecode
2024-08-27 13:32:33 +02:00 · 2024-08-27 10:16:51 +02:00 · 2024-08-26 11:04:23 +02:00 · 2024-08-23 13:37:25 +02:00 · 2024-08-23 13:21:31 +02:00 · 2024-08-21 15:19:11 +02:00
512 changed files with 29610 additions and 7525 deletions
@@ -0,0 +1,30 @@
+**/.classpath
+**/.dockerignore
+**/.env
+**/.git
+**/.gitignore
+**/.project
+**/.settings
+**/.toolstarget
+**/.vs
+**/.vscode
+**/*.*proj.user
+**/*.dbmdl
+**/*.jfm
+**/azds.yaml
+**/bin
+**/charts
+**/docker-compose*
+**/Dockerfile*
+**/node_modules
+**/npm-debug.log
+**/obj
+**/secrets.dev.yaml
+**/values.dev.yaml
+LICENSE
+README.md
+!**/.gitignore
+!.git/HEAD
+!.git/config
+!.git/packed-refs
+!.git/refs/heads/**
@@ -0,0 +1,26 @@
+name: Docker - AutoClient
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/AutoClient/**'
+      - '!Tools/AutoClient/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-autoclient.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/AutoClient/docker/Dockerfile
+      docker_repo: codexstorage/codex-autoclient
+    secrets: inherit
+
@@ -0,0 +1,28 @@
+name: Docker - Discord Bot
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/BiblioTech/**'
+      - '!Tools/BiblioTech/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-discordbot.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/BiblioTech/docker/Dockerfile
+      docker_repo: codexstorage/codex-discordbot
+    secrets: inherit
+
@@ -0,0 +1,27 @@
+name: Docker - KeyMaker
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/KeyMaker/**'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-KeyMaker.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/KeyMaker/docker/Dockerfile
+      docker_repo: codexstorage/codex-keymaker
+    secrets: inherit
+
@@ -0,0 +1,27 @@
+name: Docker - MarketInsights API
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/MarketInsights/**'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-marketinsights.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/MarketInsights/Dockerfile
+      docker_repo: codexstorage/codex-marketinsights
+    secrets: inherit
+
@@ -0,0 +1,178 @@
+name: Reusable - Docker
+
+
+on:
+  workflow_call:
+    inputs:
+      docker_file:
+        default: docker/Dockerfile
+        description: Dockerfile
+        required: false
+        type: string
+      docker_repo:
+        default: codexstorage/cs-codex-dist-tests
+        description: DockerHub repository
+        required: false
+        type: string
+      tag_latest:
+        default: true
+        description: Set latest tag for Docker images
+        required: false
+        type: boolean
+      tag_sha:
+        default: true
+        description: Set Git short commit as Docker tag
+        required: false
+        type: boolean
+      tag_suffix:
+        default: ''
+        description: Suffix for Docker images tag
+        required: false
+        type: string
+
+
+env:
+  DOCKER_FILE: ${{ inputs.docker_file }}
+  DOCKER_REPO: ${{ inputs.docker_repo }}
+  TAG_LATEST: ${{ inputs.tag_latest }}
+  TAG_SHA: ${{ inputs.tag_sha }}
+  TAG_SUFFIX: ${{ inputs.tag_suffix }}
+
+
+jobs:
+  # Build platform specific image
+  build:
+    strategy:
+      fail-fast: true
+      matrix:
+        target:
+          - os: linux
+            arch: amd64
+          - os: linux
+            arch: arm64
+        include:
+          - target:
+              os: linux
+              arch: amd64
+            builder: ubuntu-22.04
+          - target:
+              os: linux
+              arch: arm64
+            builder: buildjet-4vcpu-ubuntu-2204-arm
+
+    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
+    runs-on: ${{ matrix.builder }}
+    env:
+      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_REPO }}
+
+      - name: Docker - Set up Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Build and Push by digest
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ env.DOCKER_FILE }}
+          platforms: ${{ env.PLATFORM }}
+          push: true
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Docker - Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Docker - Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.target.arch }}
+          path: /tmp/digests
+          if-no-files-found: error
+          retention-days: 1
+
+
+  # Publish multi-platform image
+  publish:
+    name: Publish multi-platform image
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Docker - Variables
+        run: |
+          # Adjust custom suffix when set and
+          if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
+            echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
+          fi
+          # Disable SHA tags on tagged release
+          if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
+            echo "TAG_SHA=false" >>$GITHUB_ENV
+          fi
+          # Handle latest and latest-custom using raw
+          if [[ ${{ env.TAG_SHA }} == "false" ]]; then
+            echo "TAG_LATEST=false" >>$GITHUB_ENV
+            echo "TAG_RAW=true" >>$GITHUB_ENV
+            if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
+              echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
+            else
+              echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
+            fi
+          else
+            echo "TAG_RAW=false" >>$GITHUB_ENV
+          fi
+
+      - name: Docker - Download digests
+        uses: actions/download-artifact@v4
+        with:
+          pattern: digests-*
+          merge-multiple: true
+          path: /tmp/digests
+
+      - name: Docker - Set up Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_REPO }}
+          flavor: |
+            latest=${{ env.TAG_LATEST }}
+            suffix=${{ env.TAG_SUFFIX }},onlatest=true
+          tags: |
+            type=semver,pattern={{version}}
+            type=raw,enable=${{ env.TAG_RAW }},value=latest
+            type=sha,enable=${{ env.TAG_SHA }}
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
+
+      - name: Docker - Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}
@@ -0,0 +1,26 @@
+name: Docker - Rewarder Bot
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/TestNetRewarder/**'
+      - '!Tools/TestNetRewarder/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-rewarder.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/TestNetRewarder/docker/Dockerfile
+      docker_repo: codexstorage/codex-rewarderbot
+    secrets: inherit
+
@@ -0,0 +1,22 @@
+name: Docker - Runner
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - docker/Dockerfile
+      - docker/docker-entrypoint.sh
+      - .github/workflows/docker-runner.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    secrets: inherit
@@ -1,123 +0,0 @@
-name: Docker
-
-
-on:
-  push:
-    branches:
-      - master
-    tags:
-      - 'v*.*.*'
-    paths:
-      - docker/Dockerfile
-      - docker/docker-entrypoint.sh
-      - .github/workflows/docker.yml
-  workflow_dispatch:
-
-
-env:
-  DOCKER_FILE: docker/Dockerfile
-  DOCKER_REPO: codexstorage/cs-codex-dist-tests
-
-
-jobs:
-  # Build platform specific image
-  build:
-    strategy:
-      fail-fast: true
-      matrix:
-        target:
-          - os: linux
-            arch: amd64
-          - os: linux
-            arch: arm64
-        include:
-          - target:
-              os: linux
-              arch: amd64
-            builder: ubuntu-22.04
-          - target:
-              os: linux
-              arch: arm64
-            builder: buildjet-4vcpu-ubuntu-2204-arm
-
-    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
-    runs-on: ${{ matrix.builder }}
-    outputs:
-      tags-linux-amd64: ${{ steps.tags.outputs.tags-linux-amd64 }}
-      tags-linux-arm64: ${{ steps.tags.outputs.tags-linux-arm64 }}
-    env:
-      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
-      SUFFIX: ${{ format('{0}-{1}', 'linux', matrix.target.arch) }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.DOCKER_REPO }}
-          flavor: |
-            latest=false
-          tags: |
-            type=semver,pattern={{version}},suffix=-${{ env.SUFFIX }}
-            type=sha,suffix=-${{ env.SUFFIX }},enable=${{ !startsWith(github.ref, 'refs/tags/') }}
-
-      - name: Docker - Set tags output
-        id: tags
-        run: |
-          if [[ '${{ matrix.target.os }}' == 'linux' && '${{ matrix.target.arch }}' == 'amd64' ]]; then
-            echo "tags-linux-amd64=${{ steps.meta.outputs.tags }}" >> "$GITHUB_OUTPUT"
-          elif [[ '${{ matrix.target.os }}' == 'linux' && '${{ matrix.target.arch }}' == 'arm64' ]]; then
-            echo "tags-linux-arm64=${{ steps.meta.outputs.tags }}" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Build and Push
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ${{ env.DOCKER_FILE }}
-          platforms: ${{ env.PLATFORM }}
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-
-  # Publish single image
-  publish:
-    name: Push single image
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.DOCKER_REPO }}
-          tags: |
-            type=semver,pattern={{version}}
-            type=sha,enable=${{ !startsWith(github.ref, 'refs/tags/') }}
-
-      - name: Docker - Set tags
-        run: |
-          # Transform multi-line tags in to the comma-seperated
-          TAGS=$(echo "${{ steps.meta.outputs.tags }}" | tr '\n' ',' | awk '{gsub(/,$/,"");}1')
-          echo "TAGS=${TAGS}" >>$GITHUB_ENV
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Create and push manifest images
-        uses: Noelware/docker-manifest-action@master
-        with:
-          inputs: ${{ env.TAGS }}
-          images: ${{ needs.build.outputs.tags-linux-amd64 }},${{ needs.build.outputs.tags-linux-arm64 }}
-          push: true
@@ -0,0 +1,20 @@
+name: Report - HoldMyBeerTest
+
+
+on:
+  # schedule:
+  #   - cron: '30 1 */2 * *'
+  workflow_dispatch:
+
+jobs:
+  run_tests:
+    name: Call runner
+    uses: ./.github/workflows/run-continuous-tests.yaml
+    with:
+      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+      branch: master
+      nameprefix: c-tests-report-holdmybeertest-48h
+      tests_target_duration: 48h
+      tests_filter: HoldMyBeerTest
+      tests_cleanup: true
+    secrets: inherit
@@ -0,0 +1,20 @@
+name: Report - PeersTest
+
+
+on:
+  # schedule:
+  #   - cron: '30 1 */2 * *'
+  workflow_dispatch:
+
+jobs:
+  run_tests:
+    name: Call runner
+    uses: ./.github/workflows/run-continuous-tests.yaml
+    with:
+      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+      branch: master
+      nameprefix: c-tests-report-peerstest-48h
+      tests_target_duration: 48h
+      tests_filter: PeersTest
+      tests_cleanup: true
+    secrets: inherit
@@ -0,0 +1,170 @@
+name: Run Continuous Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
+        required: false
+        type: string
+      nameprefix:
+        description: Resources prefix (c-tests)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (2d)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup
+        required: false
+        type: boolean
+        default: true
+  workflow_call:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
+        required: false
+        type: string
+      nameprefix:
+        description: Resources prefix (c-tests)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (2d)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup
+        required: false
+        type: boolean
+        default: true
+      workflow_source:
+        description: Workflow source
+        required: false
+        type: string
+        default: ''
+
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
+  NAMEPREFIX: c-tests
+  NAMESPACE: default
+  TESTS_TARGET_DURATION: 2d
+  TESTS_FILTER: ""
+  TESTS_CLEANUP: true
+  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.28.2
+
+
+jobs:
+  run_tests:
+    name: Run Continuous Tests ${{ inputs.tests_filter }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
+          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          echo "DEPLOYMENT_NAMESPACE=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}"
+          echo "Runner pod: `kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}'`"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+          echo "Tests runid: ${{ env.RUNID }}"
+          echo "Tests codexdockerimage: ${{ env.CODEXDOCKERIMAGE }}"
+          echo "Tests namespace: ${{ env.DEPLOYMENT_NAMESPACE }}"
+          echo "Tests duration: ${{ env.TESTS_TARGET_DURATION }}"
+          echo "Tests filter: ${{ env.TESTS_FILTER }}"
+          echo "Tests cleanup: ${{ env.TESTS_CLEANUP }}"
+          echo "----"
+
+      - name: Show Runner logs
+        if: false
+        run: |
+          # Variables
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          duration=600
+          namespace="${{ env.NAMESPACE }}"
+          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")
+
+          # Check Pod status
+          WAIT=120
+          SECONDS=0
+          sleep=1
+          while (( SECONDS < WAIT )); do
+            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath="{.status.phase}")
+            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in $phase state - Get the logs"; break; } || { echo "Pod $pod is in $phase state - Retry in $sleep seconds / $((WAIT - SECONDS))"; }
+            sleep $sleep
+          done
+
+          # Get logs
+          timeout $duration \
+            kubectl logs $pod \
+              -n $namespace \
+              -f \
+              --tail=-1 \
+              --timestamps || true
@@ -1,45 +1,40 @@
-name: Dist Tests
+name: Run Dist Tests


 on:
-  # push:
-  #   branches:
-  #     - master
-  #   tags:
-  #     - 'v*.*.*'
-  #   paths-ignore:
-  #     - '**/*.md'
-  #     - '.gitignore'
-  #     - 'docker/**'
-  #     - '!docker/job.yaml'
  workflow_dispatch:
    inputs:
-      branch:
-        description: Branch (master)
-        required: false
-        type: string
      source:
        description: Repository with tests (current)
        required: false
        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
      nameprefix:
-        description: Runner prefix (cs-codex-dist-tests)
+        description: Runner name prefix (d-tests-runner)
        required: false
        type: string
      namespace:
-        description: Runner namespace (cs-codex-dist-tests)
+        description: Runner namespace (default)
+        required: false
+        type: string
+      command:
+        description: Command (dotnet test Tests/CodexTests)
        required: false
        type: string


 env:
-  BRANCH: ${{ github.ref_name }}
  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
-  NAMEPREFIX: cs-codex-dist-tests
-  NAMESPACE: cs-codex-dist-tests
-  JOB_MANIFEST: docker/job.yaml
+  BRANCH: ${{ github.ref_name }}
+  NAMEPREFIX: d-tests-runner
+  NAMESPACE: default
+  COMMAND: dotnet test Tests/CodexTests
+  JOB_MANIFEST: docker/dist-tests-job.yaml
  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
-  KUBE_VERSION: v1.26.1
+  KUBE_VERSION: v1.28.2


 jobs:
@@ -48,19 +43,21 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4

      - name: Variables
        run: |
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'$COMMAND'"')" >>"$GITHUB_ENV"
+          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
          [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
-          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
-          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"

      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
-        uses: azure/setup-kubectl@v3
+        uses: azure/setup-kubectl@v4
        with:
          version: ${{ env.KUBE_VERSION }}

@@ -72,3 +69,15 @@ jobs:
      - name: Kubectl - Create Job
        run: |
          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+          echo "Tests runid: ${{ env.RUNID }}"
+          echo "Tests command: `jq -r '. | join(" ")' <<< '${{ env.COMMAND }}'`"
+          echo "----"
@@ -1,4 +1,5 @@
 .vs
 obj
 bin
-.vscode
+.vscode
+Tools/AutoClient/datapath
@@ -1,254 +0,0 @@
-using System.Reflection;
-
-namespace ArgsUniform
-{
-    public class ArgsUniform<T>
-    {
-        private readonly Action printAppInfo;
-        private readonly object? defaultsProvider;
-        private readonly IEnv.IEnv env;
-        private readonly string[] args;
-        private const int cliStart = 8;
-        private const int shortStart = 38;
-        private const int envStart = 48;
-        private const int descStart = 80;
-
-        public ArgsUniform(Action printAppInfo, params string[] args)
-            : this(printAppInfo, new IEnv.Env(), args)
-        {
-        }
-
-        public ArgsUniform(Action printAppInfo, object defaultsProvider, params string[] args)
-            : this(printAppInfo, defaultsProvider, new IEnv.Env(), args)
-        {
-        }
-
-        public ArgsUniform(Action printAppInfo, IEnv.IEnv env, params string[] args)
-            : this(printAppInfo, null!, env, args)
-        {
-        }
-
-        public ArgsUniform(Action printAppInfo, object defaultsProvider, IEnv.IEnv env, params string[] args)
-        {
-            this.printAppInfo = printAppInfo;
-            this.defaultsProvider = defaultsProvider;
-            this.env = env;
-            this.args = args;
-        }
-
-        public T Parse(bool printResult = false)
-        {
-            if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
-            {
-                printAppInfo();
-                PrintHelp();
-                throw new Exception();
-            }
-
-            var result = Activator.CreateInstance<T>();
-            var uniformProperties = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
-            var missingRequired = new List<PropertyInfo>();
-            foreach (var uniformProperty in uniformProperties)
-            {
-                var attr = uniformProperty.GetCustomAttribute<UniformAttribute>();
-                if (attr != null)
-                {
-                    if (!UniformAssign(result, attr, uniformProperty) && attr.Required)
-                    {
-                        {
-                            missingRequired.Add(uniformProperty);
-                        }
-                    }
-                }
-            }
-
-            if (missingRequired.Any())
-            {
-                PrintResults(result, uniformProperties);
-                Print("");
-                foreach (var missing in missingRequired)
-                {
-                    var attr = missing.GetCustomAttribute<UniformAttribute>()!;
-                    var exampleArg = $"--{attr.Arg}=...";
-                    var exampleEnvVar = $"{attr.EnvVar}=...";
-                    Print($" ! Missing required input. Use argument: '{exampleArg}' or environment variable: '{exampleEnvVar}'.");
-                }
-
-                PrintHelp();
-                throw new ArgumentException("Unable to assemble all required arguments");
-            }
-
-            if (printResult)
-            {
-                PrintResults(result, uniformProperties);
-            }
-
-            return result;
-        }
-
-        private void PrintResults(T result, PropertyInfo[] uniformProperties)
-        {
-            Print("");
-            foreach (var p in uniformProperties)
-            {
-                Print($"\t{p.Name} = {p.GetValue(result)}");
-            }
-            Print("");
-        }
-
-        public void PrintHelp()
-        {
-            Print("");
-            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description");
-            var attrs = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).Select(p => p.GetCustomAttribute<UniformAttribute>()).Where(a => a != null).ToArray();
-            foreach (var attr in attrs)
-            {
-                var a = attr!;
-                var optional = !a.Required ? " *" : "";
-                PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional);
-            }
-            Print("");
-        }
-
-        private void Print(string msg)
-        {
-            Console.WriteLine(msg);
-        }
-
-        private void PrintAligned(string cli, string s, string env, string desc)
-        {
-            Console.CursorLeft = cliStart;
-            Console.Write(cli);
-            Console.CursorLeft = shortStart;
-            Console.Write(s);
-            Console.CursorLeft = envStart;
-            Console.Write(env);
-            Console.CursorLeft = descStart;
-            Console.Write(desc + Environment.NewLine);
-        }
-
-        private object GetDefaultValue(Type t)
-        {
-            if (t.IsValueType) return Activator.CreateInstance(t)!;
-            return null!;
-        }
-
-        private bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
-            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
-            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
-            return false;
-        }
-
-        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
-        {
-            var currentValue = uniformProperty.GetValue(result);
-            var isEmptryString = (currentValue as string) == string.Empty;
-            if (currentValue != GetDefaultValue(uniformProperty.PropertyType) && !isEmptryString) return true;
-
-            if (defaultsProvider == null) return false;
-
-            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
-            if (defaultProperty == null) return false;
-
-            var value = defaultProperty.GetValue(defaultsProvider);
-            if (value != null)
-            {
-                return Assign(result, uniformProperty, value);
-            }
-            return false;
-        }
-
-        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
-            if (!string.IsNullOrEmpty(e))
-            {
-                return Assign(result, uniformProperty, e);
-            }
-            return false;
-        }
-
-        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
-        {
-            var fromArg = GetFromArgs(attr.Arg);
-            if (fromArg != null)
-            {
-                return Assign(result, uniformProperty, fromArg);
-            }
-            var fromShort = GetFromArgs(attr.ArgShort);
-            if (fromShort != null)
-            {
-                return Assign(result, uniformProperty, fromShort);
-            }
-            return false;
-        }
-
-        private bool Assign(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (uniformProperty.PropertyType == value.GetType())
-            {
-                uniformProperty.SetValue(result, value);
-                return true;
-            }
-            else
-            {
-                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
-                {
-                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
-                    return true;
-                }
-                else
-                {
-                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
-                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
-                    
-                    throw new NotSupportedException();
-                }
-            }
-        }
-
-        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
-        {
-            var s = value.ToString();
-            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
-            {
-                uniformProperty.SetValue(result, e);
-                return true;
-            }
-            return false;
-        }
-
-        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (int.TryParse(value.ToString(), out int i))
-            {
-                uniformProperty.SetValue(result, i);
-                return true;
-            }
-            return false;
-        }
-
-        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
-        {
-            if (value != null)
-            {
-                uniformProperty.SetValue(result, true);
-            }
-            return true;
-        }
-
-        private string? GetFromArgs(string key)
-        {
-            var argKey = $"--{key}=";
-            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
-            if (arg != null)
-            {
-                return arg.Substring(argKey.Length);
-            }
-            return null;
-        }
-    }
-}
@@ -0,0 +1,158 @@
+# Distributed System Tests for Nim-Codex
+
+## Contributing plugins
+The testing framework was created for testing Codex. However, it's been designed such that other containerized projects can 'easily' be added.
+
+In this file, you'll see 'users' (in quote) mentioned once or twice. This refers to code/projects/tests which end up making use of your plugin. 'Users' come in many shapes and sizes and tend to have many differen use-cases in mind. Please consider this when reading this document and writing your plugin.
+
+## Checklist
+Your application must pass this checklist to be compatible with the framework:
+- It runs in a docker container.
+- It can be configured via environment variables. (You may need to create a docker image which contains a shell script, to pass some env-vars as CLI arguments to your application. Container command overrides do work, but are not equally reliable across container platforms. When in doubt: use env-var!)
+- It has network interaction:
+   - It exposes one or more APIs via one or more ports, OR
+   - It makes calls to other services. (OR both.)
+
+If your application's use-cases rely primarily on shell interaction, this framework might not be for you. The framework allows you to execute commands in containers AND read stdout/stderr responses. However, its focus during development has always been webservice API interactions.
+
+## Steps
+In order to add your project to the framework you must:
+1. Create a library assembly in the project plugins folder.
+1. It must contain a type that implements the `IProjectPlugin` interface from the `Core` assembly.
+1. If your plugin wants to expose any specific methods or objects to 'users', it must implement extensions for the `CoreInterface` type.
+1. If your plugin wants to run containers of its own project, it must provide a recipe.
+
+## Constructors & Tools
+Your implementation of `IProjectPlugin` must have a public constructor with a single argument of type `IPluginTools`, for example:
+```C#
+   public class MyPlugin : IProjectPlugin
+   {
+      public MyPlugin(IPluginTools tools)
+      {
+         ...
+      }
+
+      ...
+   }
+```
+
+## Plugin Interfaces
+The `IProjectPlugin` interface requires the implementation of two methods.
+1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version and/or configuration information at this time if applicable.
+1. `Decommission` - Should your plugin have any active system resources, free them in this method. Please note that resources managed by the framework (such as running containers and tracked data files) do *not* need to be manually disposed in this method. `Decommission` is to be used for resources not managed by the framework.
+
+There are a few optional interfaces your plugin may choose to implement. The framework will automatically use these interfaces.
+1. `IHasLogPrefix` - Implementing this interface allows you to provide a string which will be prepended to all log statements made by your plugin. A polite thing to do.
+1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by 'users' of your plugin. Often this data finds its way into log files and container-descriptors in order to help track versions/tests/deployments, etc.
+
+## IPluginTools
+`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients to make calls to containers. (Figure out addresses and ports for containers is handled by the framework.)
+
+It is possible and allowed for your plugin to depend on and use other plugins. (For example, maybe your project wants to interact with Ethereum and wants to use the GethPlugin to talk to a Geth node.) `IPluginTools` is *not* what is used for accessing functionality of other plugins. See 'Core Interface' section.
+
+ILog GetLog();
+IHttp CreateHttp(Action<HttpClient> onClientCreated);
+IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
+IHttp CreateHttp();
+IFileManager GetFileManager();
+
+The plugin tools provide:
+1. `Workflow` - This tool allows you to start and stop containers using "container recipes". (More on those below.) It also allows you to execute commands inside a container, access stdout/stderr, detect crashes, and access pod deployment information. The workflow tool also lets you inspect the locations available in the cluster, and decide where you want to run containers. (More on that below as well.)
+1. `Log` - Good logging is priceless. Use this tool to get a log object handle, and write useful debug/info/error statements.
+1. `Http` - This tool gives you a convenient way to access a standard dotnet HttpClient, and takes care of timeouts and retries (in accordance with the config). Additionally, it combos nicely with container objects created by `Workflow`, such that you never have to spend any time figuring out the addresses and ports of your containers.
+1. `FileManager` - Lets you use tracked temporary files. Even if the 'user' tests/application start crashing, the framework will make sure these are cleaned up.
+
+## Core Interface
+Any functionality your plugin wants to expose to 'users' will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type:
+```C#
+   public static class CoreInterfaceExtensions
+   {
+      public static MyPluginReturnType DoSomethingCool(this CoreInterface ci, string someArgument)
+      {
+         return Plugin(ci).SomethingCool(someArgument);
+      }
+
+      private static MyPlugin Plugin(CoreInterface ci)
+      {
+         return ci.GetPlugin<MyPlugin>();
+      }
+   }
+```
+
+If your plugin wants to access the functionality exposed by other plugins, then you can pass the argument `CoreInterface ci` to your plugin code in order to do so. (For example, if you want to start a Geth node, the Geth plugin adds `IGethNode StartGethNode(this CoreInterface ci, Action<IGethSetup> setup)` to the core interface.) Don't forget you'll need to add a project reference to each plugin project you wish to use.
+
+While technically you can build whatever you like on top of the `CoreInterface` and your own plugin types, I recommend that you follow the approach explained below.
+
+## Deploying, Wrapping, and Starting
+When building a plugin, it is important to make as few assumptions as possible about how it will be used by whoever is going to use the framework. For this reason, I recommend you expose three kinds of methods using your `CoreInterface` extensions:
+1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning container objects as a result. If your project requires additional information, you can create a new class type to contain both it and the container objects created.
+1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain. (For example, if my project has a REST API call that allows users to fetch some state information, the object returned by Wrap should have a convenient method to call that API and receive that state information.)
+1. Start - This kind of method does both, simply calling a Deploy method first, then a Wrap method, and returns the result.
+
+Here's an example:
+```C#
+public static class CoreInterfaceExtensions
+   {
+      public static RunningContainers DeployMyProject(this CoreInterface ci, string someArgument)
+      {
+         // `RunningContainers` is a framework type. It contains all necessary information about a deployed container. It is serializable.
+         // Should you need to return any additional information, create a new type that contains it as well as the container information. Make sure it is serializable.
+         return Plugin(ci).DeployMyProjectContainer(someArgument); // <-- This method should use the `PluginTools.CreateWorkflow()` tool to deploy a container with a configuration that matches someArguments.
+      }
+
+      public static IMyProjectNode WrapMyProjectContainer(this CoreInterface ci, RunningContainers container)
+      {
+         return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` to create an HTTP client for the container, then wrap it in an object that
+         // represents the API of your project, in this case 'IMyProjectNode'.
+      }
+
+      public static IMyProjectNode StartMyProject(this CoreInterface ci, string someArgument)
+      {
+         // Start is now nothing more than a convenience method, combining the previous two.
+         var rc = ci.DeployMyProject(someArgument);
+         return WrapMyProjectContainer(ci, rc);
+      }
+   }
+```
+
+Should your deploy methods not return framework-types like RunningContainers, please make sure that your custom times are serializable. (Decorate them with the `[Serializable]` attribute.) Tools have been built using this framework which rely on the ability to serialize and store deployment information for later use. Please don't break this possibility. (Consider using the `SerializeGate` type to help ensure compatibility.)
+
+The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinuousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing.
+
+## Container Recipes
+In order to run a container of your application, the framework needs to know how to create that container. Think of a container recipe as being similar to a docker-compose.yaml file: You specify the docker image, ports, environment variables, persistent volumes, and secrets. However, container recipes are code. This allows you to add conditional behaviour to how your container is constructed. For example: The 'user' of your plugin specifies in their call input that they want to run your application in a certain mode. This would cause your container recipe to set certain environment variables, which cause the application to behave in the requested way.
+
+### Addresses and ports
+In a docker-compose.yaml file, it is perfectly normal to specify which ports should be exposed on your container. However, in the framework there's more to consider. When your application container starts, who knows on what kind of machine it runs, and what other processes it's sharing space with? Well, Kubernetes knows. Therefore, it is recommended that container recipes *do not* specify exact port numbers. The framework allows container recipes to declare "a port" without specifying its port number. This allows the framework and Kubernetes to figure out which ports are available when it's time to deploy. In order to find out which port numbers were assigned post-deployment, you can look up the port by tag (which is just an identifying string). When you specify a port to be mapped in your container recipe, you must specify:
+1. `Tag` - An identifier.
+1. `Internal` or `External` - Whether this port should be accessible only inside the cluster (for other containers (k8s: "ClusterIP")) or outside the cluster as well (for external tools/applications (k8s: "NodePort")).
+1. `Protocol` - TCP or UDP. Both protocols on the same port is not universally supported by all container engines, and is therefore not supported by the framework.
+
+If your application wants to listen for incoming traffic from inside its container, be sure to bind it to address "0.0.0.0".
+
+Reminder: If you don't want to worry about addresses, and internal or external ports, you don't have to! The container objects returned by the `workflow` plugin tool have a method called `GetAddress`. Given a port tag, it returns and address object. The `Http` plugin tool can use that address object to set up connections.
+
+## Locations
+The framework is designed to allow you to control instances of your application in multiple (physical) locations. It accomplishes this by using kubernetes, and the ability to deploy containers to specific hosts (nodes) inside a kubernetes cluster. Since Kubernetes allows you to build clusters cross-site, this framework in theory enables you to deploy and interact with containers running anywhere.
+
+The `workflow` plugin tool provides you a list of all available locations in the cluster. When starting a container, you are able to pick one of those locations. If no location is selected, one will be chosen by kubernetes. Locations can be chosen explicitly by kubernetes node name, or, they can be picked from the array of available locations.
+
+Example:
+```C#
+{
+   var location = Ci.GetKnownLocations().Get("kbnode_euwest_paris1");
+   var codex = Ci.StartCodexNode(s => s.At(location));
+}
+```
+In this example, 'Ci' is an instance of the core interface. The CodexPlugin exposes a function 'StartCodexNode', which allows its user to specify a location. This location is then passed to the `workflow` tool when the Codex plugin starts its container.
+
+The available locations array guarantees that each entry corresponds to a different kubernetes host.
+```C#
+{
+   var knownLocations = Ci.GetKnownLocations();
+   // I don't care where exactly, as long as they are different locations.
+   var codexAtZero = Ci.StartCodexNode(s => s.At(knownLocations.Get(0)));
+   var codexAtOne = Ci.StartCodexNode(s => s.At(knownLocations.Get(1)));
+}
+```
+
@@ -1,24 +1,40 @@
 # Distributed System Tests for Nim-Codex

 ## Contributing tests
-Do you want to write some tests for Codex using this distributed test setup? Great! Here's what you do.
+Do you want to write some tests using this distributed test setup? Great! Here's what you do.

 1. Create a branch. Name it something descriptive, but start it with `tests/` please. [Example: `tests/data-redundancy`.]
-1. Checkout your branch, and decide if your tests will be 'short' tests (minutes to hours), or 'long' tests (hours to days), or both! Create a folder for your tests in the matching folders (`/Tests`, `/LongTests`) and don't worry: You can always move your tests later if you like. [Example, short: `/Tests/DataRedundancy/`, long: `/LongTests/DataRedundancy/`]
-1. Create one or more code files in your new folder, and write some tests! Here are some tips to help you get started. You can always take a look at the example tests found in [`/Tests/BasicTests/ExampleTests.cs`](/Tests/BasicTests/ExampleTests.cs)
-   1. Set up a standard NUnit test fixture.
-   1. Inherrit from `DistTest` or `AutoBootstrapDistTest`.
-   1. When using `DistTest`:
-      1. You must start your own Codex bootstrap node. You can use `SetupCodexBootstrapNode(...)` for this.
-      1. When you start other Codex nodes with `SetupCodexNodes(...)` you can pass the bootstrap node by adding the `.WithBootstrapNode(...)` option.
-   1. When using `AutoBootstrapDistTest`:
-      1. The test-infra creates the bootstrap node for you, and automatically passes it to each Codex node you create in your tests. Handy for keeping your tests clean and to-the-point.
-      1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `DistTest` instead.
-   1. You can generate files of random test data by calling `GenerateTestFile(...)`.
-   1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
-   1. You can enable access to the Codex node metrics by adding the option `.EnableMetrics()`. Enabling metrics will make the test-infra download and save all Codex metrics in case of a test failure. (The metrics are stored as CSV, in the same location as the test log file.)
-   1. You can enable access to the blockchain marketplace by adding the option `.EnableMarketplace(...)`.
-   1. Enabling metrics and/or enabling the marketplace takes extra resources from the test-infra and increases the time needed during Codex node setup. Please don't enable these features unless your tests need them.
-   1. Tip: Codex nodes can be named. Use the option `WithName(...)` and make reading your test logs a little nicer!
-   1. Tip: Commit often.
-1. Once you're happy with your tests, please create a pull-request and ask (another) Codex core contributor to review your changes.
+1. Checkout your branch.
+1. Create a new assembly in the `/Tests` folder. This can be an NUnit test assembly or simply a console app.
+1. Add Project references to `Core`, as well as any project plugin you'll be using.
+1. Write tests! Use existing tests for inspiration.
+
+## Tips for writing tests for Codex
+### Transient tests
+1. Add new code files to `Tests/CodexTests`
+1. Inherrit from `CodexDistTest` or `AutoBootstrapDistTest`.
+1. When using `CodexDistTest`:
+   1. You must start your own Codex bootstrap node. You can use `AddCodex(...)` for this.
+   1. When you start other Codex nodes with `AddCodex(...)` you can pass the bootstrap node by adding the `.WithBootstrapNode(...)` option.
+1. When using `AutoBootstrapDistTest`:
+   1. The test-infra creates the bootstrap node for you, and automatically passes it to each Codex node you create in your tests. Handy for keeping your tests clean and to-the-point.
+   1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `CodexDistTest` instead.
+1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
+### Continuous tests
+1. Add new code files to `Tests/CodexContinuousTests/Tests`
+1. Inherrit from `ContinuousTest`
+1. Define one or more methods and decorate them with the `[TestMoment(...)]` attribute.
+1. The TestMoment takes a number of seconds as argument. Each moment will be executed by the continuous test runner applying the given seconds as delay. (Non-cumulative. So two moments at T:10 will be executed one after another without delay, in this case the order of execution should not be depended upon.)
+1. Continuous tests automatically receive access to the Codex nodes that the tests are being run against.
+1. Additionally, Continuous tests can start their own transient Codex nodes and bootstrap them against the persistent nodes.
+
+### Tips for either type of test
+1. You can generate files of random test data by calling `GenerateTestFile(...)`.
+1. You can enable access to the Codex node metrics by adding the option `.EnableMetrics()`. Enabling metrics will make the test-infra download and save all Codex metrics in case of a test failure. (The metrics are stored as CSV, in the same location as the test log file.)
+1. You can enable access to the blockchain marketplace by adding the option `.EnableMarketplace(...)`.
+1. Enabling metrics and/or enabling the marketplace takes extra resources from the test-infra and increases the time needed during Codex node setup. Please don't enable these features unless your tests need them.
+1. Tip: Codex nodes can be named. Use the option `WithName(...)` and make reading your test logs a little nicer!
+1. Tip: Commit often.
+
+## Don't forget
+1. Once you're happy with your tests, please create a pull-request and ask a Codex core contributor to review your changes.
@@ -1,97 +0,0 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using DistTestCore.Marketplace;
-using KubernetesWorkflow;
-
-namespace CodexNetDeployer
-{
-    public class CodexNodeStarter
-    {
-        private readonly Configuration config;
-        private readonly TestLifecycle lifecycle;
-        private readonly GethStartResult gethResult;
-        private string bootstrapSpr = "";
-        private int validatorsLeft;
-
-        public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
-        {
-            this.config = config;
-            this.lifecycle = lifecycle;
-            this.gethResult = gethResult;
-            validatorsLeft = numberOfValidators;
-        }
-
-        public RunningContainer? Start(int i)
-        {
-            Console.Write($" - {i} = ");
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
-            var workflowStartup = new StartupConfig();
-            workflowStartup.Add(gethResult);
-            workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));
-
-            var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
-
-            var container = containers.Containers.First();
-            var codexAccess = new CodexAccess(lifecycle.Log, container, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(container));
-            var account = gethResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[i];
-            var tokenAddress = gethResult.MarketplaceNetwork.Marketplace.TokenAddress;
-            var marketAccess = new MarketplaceAccess(lifecycle, gethResult.MarketplaceNetwork, account, codexAccess);
-
-            try
-            {
-                var debugInfo = codexAccess.GetDebugInfo();
-                if (!string.IsNullOrWhiteSpace(debugInfo.spr))
-                {
-                    Console.Write("Online\t");
-
-                    var interaction = gethResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
-                    interaction.MintTestTokens(new[] { account.Account }, config.InitialTestTokens, tokenAddress);
-                    Console.Write("Tokens minted\t");
-
-                    var response = marketAccess.MakeStorageAvailable(
-                        totalSpace: config.StorageSell!.Value.MB(),
-                        minPriceForTotalSpace: config.MinPrice.TestTokens(),
-                        maxCollateral: config.MaxCollateral.TestTokens(),
-                        maxDuration: TimeSpan.FromSeconds(config.MaxDuration));
-
-                    if (!string.IsNullOrEmpty(response))
-                    {
-                        Console.Write("Storage available\tOK" + Environment.NewLine);
-
-                        if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
-                        validatorsLeft--;
-                        return container;
-                    }
-                }
-            }
-            catch (Exception ex)
-            {
-                Console.WriteLine("Exception:" + ex.ToString());
-            }
-
-            Console.Write("Unknown failure. Downloading container log." + Environment.NewLine);
-            lifecycle.DownloadLog(container);
-
-            return null;
-        }
-
-        private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft)
-        {
-            var codexStart = new CodexStartupConfig(config.CodexLogLevel);
-
-            if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr;
-            codexStart.StorageQuota = config.StorageQuota!.Value.MB();
-            var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
-            marketplaceConfig.AccountIndexOverride = i;
-            codexStart.MarketplaceConfig = marketplaceConfig;
-            codexStart.MetricsEnabled = config.RecordMetrics;
-
-            if (config.BlockTTL != Configuration.SecondsIn1Day)
-            {
-                codexStart.BlockTTL = config.BlockTTL;
-            }
-
-            return codexStart;
-        }
-    }
-}
@@ -1,100 +0,0 @@
-using ArgsUniform;
-using DistTestCore.Codex;
-
-namespace CodexNetDeployer
-{
-    public class Configuration
-    {
-        public const int SecondsIn1Day = 24 * 60 * 60;
-
-        [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
-        public string KubeConfigFile { get; set; } = "null";
-
-        [Uniform("kube-namespace", "kn", "KUBENAMESPACE", true, "Kubernetes namespace to be used for deployment.")]
-        public string KubeNamespace { get; set; } = string.Empty;
-
-        [Uniform("nodes", "n", "NODES", true, "Number of Codex nodes to be created.")]
-        public int? NumberOfCodexNodes { get; set; }
-
-        [Uniform("validators", "v", "VALIDATORS", true, "Number of Codex nodes that will be validating.")]
-        public int? NumberOfValidators { get; set; }
-
-        [Uniform("storage-quota", "sq", "STORAGEQUOTA", true, "Storage quota in megabytes used by each Codex node.")]
-        public int? StorageQuota { get; set; }
-
-        [Uniform("storage-sell", "ss", "STORAGESELL", true, "Number of megabytes of storage quota to make available for selling.")]
-        public int? StorageSell { get; set; }
-
-        [Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")]
-        public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug;
-
-        [Uniform("test-tokens", "tt", "TESTTOKENS", true, "Initial amount of test-tokens minted for each Codex node.")]
-        public int InitialTestTokens { get; set; } = int.MaxValue;
-
-        [Uniform("min-price", "mp", "MINPRICE", true, "Minimum price for the storage space for which contracts will be accepted.")]
-        public int MinPrice { get; set; }
-
-        [Uniform("max-collateral", "mc", "MAXCOLLATERAL", true, "Maximum collateral that will be placed for the total storage space.")]
-        public int MaxCollateral { get; set; }
-
-        [Uniform("max-duration", "md", "MAXDURATION", true, "Maximum duration in seconds for contracts which will be accepted.")]
-        public int MaxDuration { get; set; }
-
-        [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
-        public int BlockTTL { get; set; } = SecondsIn1Day;
-
-        [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
-        public bool RecordMetrics { get; set; } = false;
-
-        [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
-            "set this option to override the label value.")]
-        public string TestsTypePodLabel { get; set; } = "continuous-tests";
-       
-        public List<string> Validate()
-        {
-            var errors = new List<string>();
-
-            ForEachProperty(
-                onString: (n, v) => StringIsSet(n, v, errors),
-                onInt: (n, v) => IntIsOverZero(n, v, errors));
-
-            if (NumberOfValidators > NumberOfCodexNodes)
-            {
-                errors.Add($"{nameof(NumberOfValidators)} ({NumberOfValidators}) may not be greater than {nameof(NumberOfCodexNodes)} ({NumberOfCodexNodes}).");
-            }
-            if (StorageSell.HasValue && StorageQuota.HasValue && StorageSell.Value >= StorageQuota.Value)
-            {
-                errors.Add("StorageSell cannot be greater than or equal to StorageQuota.");
-            }
-
-            return errors;
-        }
-
-        private void ForEachProperty(Action<string, string> onString, Action<string, int?> onInt)
-        {
-            var properties = GetType().GetProperties();
-            foreach (var p in properties)
-            {
-                if (p.PropertyType == typeof(string)) onString(p.Name, (string)p.GetValue(this)!);
-                if (p.PropertyType == typeof(int?)) onInt(p.Name, (int?)p.GetValue(this)!);
-                if (p.PropertyType == typeof(int)) onInt(p.Name, (int)p.GetValue(this)!);
-            }
-        }
-
-        private static void IntIsOverZero(string variable, int? value, List<string> errors)
-        {
-            if (value == null || value.Value < 1)
-            {
-                errors.Add($"{variable} must be set and must be greater than 0.");
-            }
-        }
-
-        private static void StringIsSet(string variable, string value, List<string> errors)
-        {
-            if (string.IsNullOrWhiteSpace(value))
-            {
-                errors.Add($"{variable} must be set.");
-            }
-        }
-    }
-}
@@ -1,111 +0,0 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using KubernetesWorkflow;
-using Logging;
-
-namespace CodexNetDeployer
-{
-    public class Deployer
-    {
-        private readonly Configuration config;
-        private readonly NullLog log;
-        private readonly DefaultTimeSet timeset;
-
-        public Deployer(Configuration config)
-        {
-            this.config = config;
-            log = new NullLog();
-            timeset = new DefaultTimeSet();
-        }
-
-        public CodexDeployment Deploy()
-        {
-            Log("Initializing...");
-            var lifecycle = CreateTestLifecycle();
-
-            Log("Preparing configuration...");
-            // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
-            var setup = new CodexSetup(999, config.CodexLogLevel);
-            setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
-            setup.MetricsEnabled = config.RecordMetrics;
-
-            Log("Creating Geth instance and deploying contracts...");
-            var gethStarter = new GethStarter(lifecycle);
-            var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
-
-            Log("Geth started. Codex contracts deployed.");
-            Log("Warning: It can take up to 45 minutes for the Geth node to finish unlocking all if its 1000 preconfigured accounts.");
-
-            // It takes a second for the geth node to unlock a single account. Let's assume 3.
-            // We can't start the codex nodes until their accounts are definitely unlocked. So
-            // We wait:
-            Thread.Sleep(TimeSpan.FromSeconds(3.0 * config.NumberOfCodexNodes!.Value));
-
-            Log("Starting Codex nodes...");
-
-            // Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
-            var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
-            var codexContainers = new List<RunningContainer>();
-            for (var i = 0; i < config.NumberOfCodexNodes; i++)
-            {
-                var container = codexStarter.Start(i);
-                if (container != null) codexContainers.Add(container);
-            }
-
-            var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers);
-
-            return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
-        }
-
-        private TestLifecycle CreateTestLifecycle()
-        {
-            var kubeConfig = GetKubeConfig(config.KubeConfigFile);
-
-            var lifecycleConfig = new DistTestCore.Configuration
-            (
-                kubeConfigFile: kubeConfig,
-                logPath: "null",
-                logDebug: false,
-                dataFilesPath: "notUsed",
-                codexLogLevel: config.CodexLogLevel,
-                k8sNamespacePrefix: config.KubeNamespace
-            );
-
-            return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
-        }
-
-        private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
-        {
-            if (!setup.MetricsEnabled) return null;
-
-            Log("Starting metrics service...");
-            var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
-            return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
-        }
-
-        private string? GetKubeConfig(string kubeConfigFile)
-        {
-            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
-            return kubeConfigFile;
-        }
-
-        private DeploymentMetadata CreateMetadata()
-        {
-            return new DeploymentMetadata(
-                kubeNamespace: config.KubeNamespace,
-                numberOfCodexNodes: config.NumberOfCodexNodes!.Value,
-                numberOfValidators: config.NumberOfValidators!.Value,
-                storageQuotaMB: config.StorageQuota!.Value,
-                codexLogLevel: config.CodexLogLevel,
-                initialTestTokens: config.InitialTestTokens,
-                minPrice: config.MinPrice,
-                maxCollateral: config.MaxCollateral,
-                maxDuration: config.MaxDuration);
-        }
-
-        private void Log(string msg)
-        {
-            Console.WriteLine(msg);
-        }
-    }
-}
@@ -1,13 +0,0 @@
-dotnet run \
-    --kube-config=/opt/kubeconfig.yaml \
-    --kube-namespace=codex-continuous-tests \
-    --nodes=5 \
-    --validators=3 \
-    --log-level=Trace \
-    --storage-quota=2048 \
-    --storage-sell=1024 \
-    --min-price=1024 \
-    --max-collateral=1024 \
-    --max-duration=3600000 \
-    --block-ttl=300 \
-    --record-metrics=true
@@ -1,19 +0,0 @@
-using ArgsUniform;
-using DistTestCore.Codex;
-
-namespace CodexNetDownloader
-{
-    public class Configuration
-    {
-        [Uniform("output-path", "o", "OUTPUT", true, "Path where files will be written.")]
-        public string OutputPath { get; set; } = "output";
-
-        [Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")]
-        public string CodexDeploymentJson { get; set; } = string.Empty;
-
-        [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
-        public string KubeConfigFile { get; set; } = "null";
-
-        public CodexDeployment CodexDeployment { get; set; } = null!;
-    }
-}
@@ -1,48 +0,0 @@
-using ArgsUniform;
-using ContinuousTests;
-using DistTestCore;
-using DistTestCore.Codex;
-using Logging;
-using Newtonsoft.Json;
-
-public class Program
-{
-    public static void Main(string[] args)
-    {
-        var nl = Environment.NewLine;
-        Console.WriteLine("CodexNetDownloader" + nl);
-
-        var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(PrintHelp, args);
-        var config = uniformArgs.Parse(true);
-
-        config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
-
-        if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
-
-        var k8sFactory = new K8sFactory();
-        var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
-
-        foreach (var container in config.CodexDeployment.CodexContainers)
-        {
-            lifecycle.DownloadLog(container);
-        }
-
-        Console.WriteLine("Done!");
-    }
-
-    private static CodexDeployment ParseCodexDeploymentJson(string filename)
-    {
-        var d = JsonConvert.DeserializeObject<CodexDeployment>(File.ReadAllText(filename))!;
-        if (d == null) throw new Exception("Unable to parse " + filename);
-        return d;
-    }
-
-    private static void PrintHelp()
-    {
-        var nl = Environment.NewLine;
-        Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
-
-        Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
-            "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
-    }
-}
@@ -1,20 +0,0 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using KubernetesWorkflow;
-using Logging;
-
-namespace ContinuousTests
-{
-    public class CodexAccessFactory
-    {
-        public CodexAccess[] Create(Configuration config, RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
-        {
-            return containers.Select(container =>
-            {
-                var address = container.ClusterExternalAddress;
-                if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
-                return new CodexAccess(log, container, timeSet, address);
-            }).ToArray();
-        }
-    }
-}
@@ -1,99 +0,0 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using KubernetesWorkflow;
-
-namespace ContinuousTests
-{
-    public class ContinuousLogDownloader
-    {
-        private readonly TestLifecycle lifecycle;
-        private readonly CodexDeployment deployment;
-        private readonly string outputPath;
-        private readonly CancellationToken cancelToken;
-
-        public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken)
-        {
-            this.lifecycle = lifecycle;
-            this.deployment = deployment;
-            this.outputPath = outputPath;
-            this.cancelToken = cancelToken;
-        }
-
-        public void Run()
-        {
-            while (!cancelToken.IsCancellationRequested)
-            {
-                UpdateLogs();
-
-                cancelToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(15));
-            }
-
-            // After testing has stopped, we wait a little bit and fetch the logs one more time.
-            // If our latest fetch was not recent, interesting test-related log activity might
-            // not have been captured yet.
-            Thread.Sleep(TimeSpan.FromSeconds(10));
-            UpdateLogs();
-        }
-
-        private void UpdateLogs()
-        {
-            foreach (var container in deployment.CodexContainers)
-            {
-                UpdateLog(container);
-            }
-        }
-
-        private void UpdateLog(RunningContainer container)
-        {
-            var filepath = Path.Combine(outputPath, GetLogName(container));
-            if (!File.Exists(filepath))
-            {
-                File.WriteAllLines(filepath, new[] { container.Name });
-            }
-
-            var appender = new LogAppender(filepath);
-
-            lifecycle.CodexStarter.DownloadLog(container, appender);
-        }
-
-        private static string GetLogName(RunningContainer container)
-        {
-            return container.Name
-                .Replace("<", "")
-                .Replace(">", "")
-                + ".log";
-        }
-    }
-
-    public class LogAppender : ILogHandler
-    {
-        private readonly string filename;
-
-        public LogAppender(string filename)
-        {
-            this.filename = filename;
-        }
-
-        public void Log(Stream log)
-        {
-            using var reader = new StreamReader(log);
-            var lines = File.ReadAllLines(filename);
-            var lastLine = lines.Last();
-            var recording = lines.Length < 3;
-            var line = reader.ReadLine();
-            while (line != null)
-            {
-                if (recording)
-                {
-                    File.AppendAllLines(filename, new[] { line });
-                }
-                else
-                {
-                    recording = line == lastLine;
-                }
-
-                line = reader.ReadLine();
-            }
-        }
-    }
-}
@@ -1,111 +0,0 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using Logging;
-
-namespace ContinuousTests
-{
-    public abstract class ContinuousTestLongTimeouts : ContinuousTest
-    {
-        public override ITimeSet TimeSet => new LongTimeSet();
-    }
-
-    public abstract class ContinuousTest
-    {
-        protected const int Zero = 0;
-        protected const int MinuteOne = 60;
-        protected const int MinuteFive = MinuteOne * 5;
-        protected const int HourOne = MinuteOne * 60;
-        protected const int HourThree = HourOne * 3;
-        protected const int DayOne = HourOne * 24;
-        protected const int DayThree = DayOne * 3;
-
-        private const string UploadFailedMessage = "Unable to store block";
-
-        public void Initialize(CodexAccess[] nodes, BaseLog log, FileManager fileManager, Configuration configuration, CancellationToken cancelToken)
-        {
-            Nodes = nodes;
-            Log = log;
-            FileManager = fileManager;
-            Configuration = configuration;
-            CancelToken = cancelToken;
-
-            if (nodes != null)
-            {
-                NodeRunner = new NodeRunner(Nodes, configuration, TimeSet, Log, CustomK8sNamespace, EthereumAccountIndex);
-            }
-            else
-            {
-                NodeRunner = null!;
-            }
-        }
-
-        public CodexAccess[] Nodes { get; private set; } = null!;
-        public BaseLog Log { get; private set; } = null!;
-        public IFileManager FileManager { get; private set; } = null!;
-        public Configuration Configuration { get; private set; } = null!;
-        public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
-        public CancellationToken CancelToken { get; private set; } = new CancellationToken();
-        public NodeRunner NodeRunner { get; private set; } = null!;
-
-        public abstract int RequiredNumberOfNodes { get; }
-        public abstract TimeSpan RunTestEvery { get; }
-        public abstract TestFailMode TestFailMode { get; }
-        public virtual int EthereumAccountIndex { get { return -1; } }
-        public virtual string CustomK8sNamespace { get { return string.Empty; } }
-
-        public string Name
-        {
-            get
-            {
-                return GetType().Name;
-            }
-        }
-
-        public ContentId? UploadFile(CodexAccess node, TestFile file)
-        {
-            using var fileStream = File.OpenRead(file.Filename);
-
-            var logMessage = $"Uploading file {file.Describe()}...";
-            var response = Stopwatch.Measure(Log, logMessage, () =>
-            {
-                return node.UploadFile(fileStream);
-            });
-
-            if (string.IsNullOrEmpty(response)) return null;
-            if (response.StartsWith(UploadFailedMessage)) return null;
-            
-            Log.Log($"Uploaded file. Received contentId: '{response}'.");
-            return new ContentId(response);
-        }
-
-        public TestFile DownloadFile(CodexAccess node, ContentId contentId, string fileLabel = "")
-        {
-            var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
-            var file = FileManager.CreateEmptyTestFile(fileLabel);
-            Stopwatch.Measure(Log, logMessage, () => DownloadToFile(node, contentId.Id, file));
-            Log.Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
-            return file;
-        }
-
-        private void DownloadToFile(CodexAccess node, string contentId, TestFile file)
-        {
-            using var fileStream = File.OpenWrite(file.Filename);
-            try
-            {
-                using var downloadStream = node.DownloadFile(contentId);
-                downloadStream.CopyTo(fileStream);
-            }
-            catch
-            {
-                Log.Log($"Failed to download file '{contentId}'.");
-                throw;
-            }
-        }
-    }
-
-    public enum TestFailMode
-    {
-        StopAfterFirstFailure,
-        AlwaysRunAllMoments
-    }
-}
@@ -1,80 +0,0 @@
-using DistTestCore;
-using Logging;
-
-namespace ContinuousTests
-{
-    public class ContinuousTestRunner
-    {
-        private readonly K8sFactory k8SFactory = new K8sFactory();
-        private readonly ConfigLoader configLoader = new ConfigLoader();
-        private readonly TestFactory testFactory = new TestFactory();
-        private readonly Configuration config;
-        private readonly StartupChecker startupChecker;
-        private readonly CancellationToken cancelToken;
-
-        public ContinuousTestRunner(string[] args, CancellationToken cancelToken)
-        {
-            config = configLoader.Load(args);
-            startupChecker = new StartupChecker(config, cancelToken);
-            this.cancelToken = cancelToken;
-        }
-
-        public void Run()
-        {
-            startupChecker.Check();
-
-            var taskFactory = new TaskFactory();
-            var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "Overview");
-            overviewLog.Log("Continuous tests starting...");
-            var allTests = testFactory.CreateTests();
-
-            ClearAllCustomNamespaces(allTests, overviewLog);
-
-            StartLogDownloader(taskFactory);
-
-            var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
-
-            foreach (var testLoop in testLoops)
-            {
-                if (cancelToken.IsCancellationRequested) break;
-
-                overviewLog.Log("Launching test-loop for " + testLoop.Name);
-                testLoop.Begin();
-                Thread.Sleep(TimeSpan.FromSeconds(5));
-            }
-
-            overviewLog.Log("Finished launching test-loops.");
-            cancelToken.WaitHandle.WaitOne();
-            overviewLog.Log("Cancelling all test-loops...");
-            taskFactory.WaitAll();
-            overviewLog.Log("All tasks cancelled.");
-        }
-
-        private void ClearAllCustomNamespaces(ContinuousTest[] allTests, FixtureLog log)
-        {
-            foreach (var test in allTests) ClearAllCustomNamespaces(test, log);
-        }
-
-        private void ClearAllCustomNamespaces(ContinuousTest test, FixtureLog log)
-        {
-            if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
-
-            log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
-            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log);
-            lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources();
-        }
-
-        private void StartLogDownloader(TaskFactory taskFactory)
-        {
-            if (!config.DownloadContainerLogs) return;
-
-            var path = Path.Combine(config.LogPath, "containers");
-            if (!Directory.Exists(path)) Directory.CreateDirectory(path);
-
-            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
-            var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
-
-            taskFactory.Run(downloader.Run);
-        }
-    }
-}
@@ -1,31 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore;
-using Logging;
-
-namespace ContinuousTests
-{
-    public class K8sFactory
-    {
-        public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log)
-        {
-            var kubeConfig = GetKubeConfig(kubeConfigFile);
-            var lifecycleConfig = new DistTestCore.Configuration
-            (
-                kubeConfigFile: kubeConfig,
-                logPath: logPath,
-                logDebug: false,
-                dataFilesPath: dataFilePath,
-                codexLogLevel: CodexLogLevel.Debug,
-                k8sNamespacePrefix: customNamespace
-            );
-
-            return new TestLifecycle(log, lifecycleConfig, timeSet, "continuous-tests", string.Empty);
-        }
-
-        private static string? GetKubeConfig(string kubeConfigFile)
-        {
-            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
-            return kubeConfigFile;
-        }
-    }
-}
@@ -1,97 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Marketplace;
-using DistTestCore;
-using KubernetesWorkflow;
-using NUnit.Framework;
-using Logging;
-using Utils;
-
-namespace ContinuousTests
-{
-    public class NodeRunner
-    {
-        private readonly K8sFactory k8SFactory = new K8sFactory();
-        private readonly CodexAccess[] nodes;
-        private readonly Configuration config;
-        private readonly ITimeSet timeSet;
-        private readonly BaseLog log;
-        private readonly string customNamespace;
-        private readonly int ethereumAccountIndex;
-
-        public NodeRunner(CodexAccess[] nodes, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex)
-        {
-            this.nodes = nodes;
-            this.config = config;
-            this.timeSet = timeSet;
-            this.log = log;
-            this.customNamespace = customNamespace;
-            this.ethereumAccountIndex = ethereumAccountIndex;
-        }
-
-        public void RunNode(Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
-        {
-            RunNode(nodes.ToList().PickOneRandom(), operation, 0.TestTokens());
-        }
-
-        public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
-        {
-            RunNode(bootstrapNode, operation, 0.TestTokens());
-        }
-
-        public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
-        {
-            var lifecycle = CreateTestLifecycle();
-            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
-
-            try
-            {
-                var debugInfo = bootstrapNode.GetDebugInfo();
-                Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
-
-                var startupConfig = new StartupConfig();
-                startupConfig.NameOverride = "TransientNode";
-                var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Trace);
-                codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
-                codexStartConfig.MarketplaceConfig.AccountIndexOverride = ethereumAccountIndex;
-                codexStartConfig.BootstrapSpr = debugInfo.spr;
-                startupConfig.Add(codexStartConfig);
-                startupConfig.Add(config.CodexDeployment.GethStartResult);
-                var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
-
-                var account = config.CodexDeployment.GethStartResult.CompanionNode.Accounts[ethereumAccountIndex];
-
-                var marketplaceNetwork = config.CodexDeployment.GethStartResult.MarketplaceNetwork;
-                if (mintTestTokens.Amount > 0)
-                {
-                    var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
-                    var interaction = marketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
-                    interaction.MintTestTokens(new[] { account.Account }, mintTestTokens.Amount, tokenAddress);
-                }
-
-                var container = rc.Containers[0];
-                var address = lifecycle.Configuration.GetAddress(container);
-                var codexAccess = new CodexAccess(log, container, lifecycle.TimeSet, address);
-                var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess);
-
-                try
-                {
-                    operation(codexAccess, marketAccess, lifecycle);
-                }
-                catch
-                {
-                    lifecycle.DownloadLog(container);
-                    throw;
-                }
-            }
-            finally
-            {
-                flow.DeleteTestResources();
-            }
-        }
-
-        private TestLifecycle CreateTestLifecycle()
-        {
-            return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log);
-        }
-    }
-}
@@ -1,227 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore;
-using Logging;
-using Utils;
-using KubernetesWorkflow;
-using NUnit.Framework.Internal;
-using System.Reflection;
-using static Program;
-
-namespace ContinuousTests
-{
-    public class SingleTestRun
-    {
-        private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
-        private readonly List<Exception> exceptions = new List<Exception>();
-        private readonly TaskFactory taskFactory;
-        private readonly Configuration config;
-        private readonly BaseLog overviewLog;
-        private readonly TestHandle handle;
-        private readonly CancellationToken cancelToken;
-        private readonly CodexAccess[] nodes;
-        private readonly FileManager fileManager;
-        private readonly FixtureLog fixtureLog;
-        private readonly string testName;
-        private readonly string dataFolder;
-
-        public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, CancellationToken cancelToken)
-        {
-            this.taskFactory = taskFactory;
-            this.config = config;
-            this.overviewLog = overviewLog;
-            this.handle = handle;
-            this.cancelToken = cancelToken;
-            testName = handle.Test.GetType().Name;
-            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
-
-            nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
-            dataFolder = config.DataPath + "-" + Guid.NewGuid();
-            fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
-        }
-
-        public void Run(EventWaitHandle runFinishedHandle)
-        {
-            taskFactory.Run(() =>
-            {
-                try
-                {
-                    RunTest();
-                    fileManager.DeleteAllTestFiles();
-                    Directory.Delete(dataFolder, true);
-                    runFinishedHandle.Set();
-                }
-                catch (Exception ex)
-                {
-                    overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex);
-                    Environment.Exit(-1);
-                }
-            });
-        }
-
-        private void RunTest()
-        {
-            try
-            {
-                RunTestMoments();
-
-                if (!config.KeepPassedTestLogs) fixtureLog.Delete();
-            }
-            catch (Exception ex)
-            {
-                fixtureLog.Error("Test run failed with exception: " + ex);
-                fixtureLog.MarkAsFailed();
-
-                if (config.StopOnFailure)
-                {
-                    OverviewLog("Configured to stop on first failure. Downloading cluster logs...");
-                    DownloadClusterLogs();
-                    OverviewLog("Log download finished. Cancelling test runner...");
-                    Cancellation.Cts.Cancel();
-                }
-            }
-        }
-
-        private void RunTestMoments()
-        {
-            var earliestMoment = handle.GetEarliestMoment();
-
-            var t = earliestMoment;
-            while (true)
-            {
-                cancelToken.ThrowIfCancellationRequested();
-
-                RunMoment(t);
-
-                if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
-                {
-                    Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping...");
-                    ThrowFailTest();
-                }
-
-                var nextMoment = handle.GetNextMoment(t);
-                if (nextMoment != null)
-                {
-                    var delta = TimeSpan.FromSeconds(nextMoment.Value - t);
-                    Log($" > Next TestMoment in {Time.FormatDuration(delta)} seconds...");
-                    cancelToken.WaitHandle.WaitOne(delta);
-                    t = nextMoment.Value;
-                }
-                else
-                {
-                    if (exceptions.Any())
-                    {
-                        ThrowFailTest();
-                    }
-                    OverviewLog(" > Test passed. " + FuturesInfo());
-                    return;
-                }
-            }
-        }
-
-        private void ThrowFailTest()
-        {
-            var ex = UnpackException(exceptions.First());
-            Log(ex.ToString());
-            OverviewLog($" > Test failed {FuturesInfo()}: " + ex.Message);
-            throw ex;
-        }
-
-        private string FuturesInfo()
-        {
-            var containers = config.CodexDeployment.CodexContainers;
-            var nodes = codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
-            var f = nodes.Select(n => n.GetDebugFutures().ToString());
-            var msg = $"(Futures: [{string.Join(", ", f)}])";
-            return msg;
-        }
-
-        private void DownloadClusterLogs()
-        {
-            var k8sFactory = new K8sFactory();
-            var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
-
-            foreach (var container in config.CodexDeployment.CodexContainers)
-            {
-                lifecycle.DownloadLog(container);
-            }
-        }
-
-        private Exception UnpackException(Exception exception)
-        {
-            if (exception is AggregateException a)
-            {
-                return UnpackException(a.InnerExceptions.First());
-            }
-            if (exception is TargetInvocationException t)
-            {
-                return UnpackException(t.InnerException!);
-            }
-
-            return exception;
-        }
-
-        private void RunMoment(int t)
-        {
-            using (var context = new TestExecutionContext.IsolatedContext())
-            {
-                try
-                {
-                    handle.InvokeMoment(t, InitializeTest);
-                }
-                catch (Exception ex)
-                {
-                    exceptions.Add(ex);
-                }
-            }
-
-            DecommissionTest();
-        }
-
-        private void InitializeTest(string name)
-        {
-            Log($" > Running TestMoment '{name}'");
-            handle.Test.Initialize(nodes, fixtureLog, fileManager, config, cancelToken);
-        }
-
-        private void DecommissionTest()
-        {
-            handle.Test.Initialize(null!, null!, null!, null!, cancelToken);
-        }
-
-        private void Log(string msg)
-        {
-            fixtureLog.Log(msg);
-        }
-
-        private void OverviewLog(string msg)
-        {
-            Log(msg);
-            var containerNames = $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
-            overviewLog.Log($"{containerNames} {testName}: {msg}");
-        }
-
-        private CodexAccess[] CreateRandomNodes(int number)
-        {
-            var containers = SelectRandomContainers(number);
-            fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
-            return codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
-        }
-
-        private RunningContainer[] SelectRandomContainers(int number)
-        {
-            var containers = config.CodexDeployment.CodexContainers.ToList();
-            var result = new RunningContainer[number];
-            for (var i = 0; i < number; i++)
-            {
-                result[i] = containers.PickOneRandom();
-            }
-            return result;
-        }
-
-        private DistTestCore.Configuration CreateFileManagerConfiguration()
-        {
-            return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
-                CodexLogLevel.Error, string.Empty);
-        }
-    }
-}
@@ -1,67 +0,0 @@
-using Logging;
-
-namespace ContinuousTests
-{
-    public class TestLoop
-    {
-        private readonly TaskFactory taskFactory;
-        private readonly Configuration config;
-        private readonly BaseLog overviewLog;
-        private readonly Type testType;
-        private readonly TimeSpan runsEvery;
-        private readonly CancellationToken cancelToken;
-        private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset);
-
-        public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, CancellationToken cancelToken)
-        {
-            this.taskFactory = taskFactory;
-            this.config = config;
-            this.overviewLog = overviewLog;
-            this.testType = testType;
-            this.runsEvery = runsEvery;
-            this.cancelToken = cancelToken;
-            Name = testType.Name;
-        }
-
-        public string Name { get; }
-
-        public void Begin()
-        {
-            taskFactory.Run(() =>
-            {
-                try
-                {
-                    while (true)
-                    {
-                        WaitHandle.WaitAny(new[] { runFinishedHandle, cancelToken.WaitHandle });
-
-                        cancelToken.ThrowIfCancellationRequested();
-
-                        StartTest();
-
-                        cancelToken.WaitHandle.WaitOne(runsEvery);
-                    }
-                }
-                catch (OperationCanceledException)
-                {
-                    overviewLog.Log("Test-loop " + testType.Name + " is cancelled.");
-                }
-                catch (Exception ex)
-                {
-                    overviewLog.Error("Test infra failure: TestLoop failed with " + ex);
-                    Environment.Exit(-1);
-                }
-            });
-        }
-
-        private void StartTest()
-        {
-            var test = (ContinuousTest)Activator.CreateInstance(testType)!;
-            var handle = new TestHandle(test);
-            var run = new SingleTestRun(taskFactory, config, overviewLog, handle, cancelToken);
-
-            runFinishedHandle.Reset();
-            run.Run(runFinishedHandle);
-        }
-    }
-}
@@ -1,60 +0,0 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using NUnit.Framework;
-
-namespace ContinuousTests.Tests
-{
-    public class ThresholdChecks : ContinuousTest
-    {
-        public override int RequiredNumberOfNodes => 1;
-        public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
-        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
-
-        private static readonly List<string> previousBreaches = new List<string>();
-
-        [TestMoment(t: 0)]
-        public void CheckAllThresholds()
-        {
-            var allNodes = CreateAccessToAllNodes();
-            foreach (var n in allNodes) CheckThresholds(n);
-        }
-
-        private void CheckThresholds(CodexAccess n)
-        {
-            var breaches = n.GetDebugThresholdBreaches();
-            if (breaches.breaches.Any())
-            {
-                var newBreaches = new List<string>();
-                foreach (var b in breaches.breaches)
-                {
-                    if (!previousBreaches.Contains(b))
-                    {
-                        newBreaches.Add(b);
-                        previousBreaches.Add(b);
-                    }
-                }
-
-                if (newBreaches.Any())
-                {
-                    Assert.Fail(string.Join(",", newBreaches.Select(b => FormatBreach(n, b))));
-
-                    Program.Cancellation.Cts.Cancel();
-                }
-            }
-        }
-
-        private string FormatBreach(CodexAccess n, string breach)
-        {
-            return $"{n.Container.Name} = '{breach}'";
-        }
-
-        private CodexAccess[] CreateAccessToAllNodes()
-        {
-            // Normally, a continuous test accesses only a subset of the nodes in the deployment.
-            // This time, we want to check all of them.
-            var factory = new CodexAccessFactory();
-            var allContainers = Configuration.CodexDeployment.CodexContainers;
-            return factory.Create(Configuration, allContainers, Log, new DefaultTimeSet());
-        }
-    }
-}
@@ -1,32 +0,0 @@
-using DistTestCore;
-using NUnit.Framework;
-
-namespace ContinuousTests.Tests
-{
-    public class TwoClientTest : ContinuousTest
-    {
-        public override int RequiredNumberOfNodes => 2;
-        public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
-        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
-
-        private ContentId? cid;
-        private TestFile file = null!;
-
-        [TestMoment(t: Zero)]
-        public void UploadTestFile()
-        {
-            file = FileManager.GenerateTestFile(10.MB());
-
-            cid = UploadFile(Nodes[0], file);
-            Assert.That(cid, Is.Not.Null);
-        }
-
-        [TestMoment(t: 10)]
-        public void DownloadTestFile()
-        {
-            var dl = DownloadFile(Nodes[1], cid!);
-
-            file.AssertIsEqual(dl);
-        }
-    }
-}
@@ -1,6 +0,0 @@
-dotnet run \
-    --kube-config=/opt/kubeconfig.yaml \
-    --codex-deployment=codex-deployment.json \
-    --keep=1 \
-    --stop=1 \
-    --dl-logs=1
@@ -1,30 +0,0 @@
-using NUnit.Framework;
-
-namespace DistTestCore
-{
-    public class AutoBootstrapDistTest : DistTest
-    {
-        public override IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
-        {
-            throw new Exception("AutoBootstrapDistTest creates and attaches a single bootstrap node for you. " +
-                "If you want to control the bootstrap node from your test, please use DistTest instead.");
-        }
-
-        public override ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
-        {
-            var codexSetup = CreateCodexSetup(numberOfNodes);
-            setup(codexSetup);
-            codexSetup.WithBootstrapNode(BootstrapNode);
-            return BringOnline(codexSetup);
-        }
-
-        [SetUp]
-        public void SetUpBootstrapNode()
-        {
-            var setup = CreateCodexSetup(1).WithName("BOOTSTRAP");
-            BootstrapNode = BringOnline(setup)[0];
-        }
-
-        protected IOnlineCodexNode BootstrapNode { get; private set; } = null!;
-    }
-}
@@ -1,42 +0,0 @@
-using Logging;
-
-namespace DistTestCore
-{
-    public class BaseStarter
-    {
-        protected readonly TestLifecycle lifecycle;
-        private Stopwatch? stopwatch;
-
-        public BaseStarter(TestLifecycle lifecycle)
-        {
-            this.lifecycle = lifecycle;
-        }
-
-        protected void LogStart(string msg)
-        {
-            Log(msg);
-            stopwatch = Stopwatch.Begin(lifecycle.Log, GetClassName());
-        }
-
-        protected void LogEnd(string msg)
-        {
-            stopwatch!.End(msg);
-            stopwatch = null;
-        }
-
-        protected void Log(string msg)
-        {
-            lifecycle.Log.Log($"{GetClassName()} {msg}");
-        }
-
-        protected void Debug(string msg)
-        {
-            lifecycle.Log.Debug($"{GetClassName()} {msg}", 1);
-        }
-
-        private string GetClassName()
-        {
-            return $"({GetType().Name})";
-        }
-    }
-}
@@ -1,92 +0,0 @@
-using KubernetesWorkflow;
-using Logging;
-using Utils;
-
-namespace DistTestCore.Codex
-{
-    public class CodexAccess
-    {
-        private readonly BaseLog log;
-        private readonly ITimeSet timeSet;
-
-        public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
-        {
-            this.log = log;
-            Container = container;
-            this.timeSet = timeSet;
-            Address = address;
-        }
-
-        public RunningContainer Container { get; }
-        public Address Address { get; }
-
-        public CodexDebugResponse GetDebugInfo()
-        {
-            return Http().HttpGetJson<CodexDebugResponse>("debug/info");
-        }
-
-        public CodexDebugPeerResponse GetDebugPeer(string peerId)
-        {
-            var http = Http();
-            var str = http.HttpGetString($"debug/peer/{peerId}");
-
-            if (str.ToLowerInvariant() == "unable to find peer!")
-            {
-                return new CodexDebugPeerResponse
-                {
-                    IsPeerFound = false
-                };
-            }
-
-            var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
-            result.IsPeerFound = true;
-            return result;
-        }
-
-        public int GetDebugFutures()
-        {
-            // Some Codex images support debug/futures to count the number of open futures.
-            return 0; // Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
-        }
-
-        public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
-        {
-            return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");
-        }
-
-        public string UploadFile(FileStream fileStream)
-        {
-            return Http().HttpPostStream("upload", fileStream);
-        }
-
-        public Stream DownloadFile(string contentId)
-        {
-            return Http().HttpGetStream("download/" + contentId);
-        }
-
-        public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
-        {
-            return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
-        }
-
-        public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
-        {
-            return Http().HttpPostJson($"storage/request/{contentId}", request);
-        }
-
-        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
-        {
-            return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
-        }
-
-        public string ConnectToPeer(string peerId, string peerMultiAddress)
-        {
-            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
-        }
-
-        private Http Http()
-        {
-            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name);
-        }
-    }
-}
@@ -1,128 +0,0 @@
-using KubernetesWorkflow;
-using Logging;
-using Newtonsoft.Json;
-using Utils;
-
-namespace DistTestCore.Codex
-{
-    public class CodexDebugResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string[] addrs { get; set; } = new string[0];
-        public string repo { get; set; } = string.Empty;
-        public string spr { get; set; } = string.Empty;
-        public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
-        public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
-        public CodexDebugVersionResponse codex { get; set; } = new();
-        public CodexDebugTableResponse table { get; set; } = new();
-    }
-
-    public class CodexDebugFutures
-    {
-        public int futures { get; set; }
-    }
-
-    public class CodexDebugTableResponse
-    {
-        public CodexDebugTableNodeResponse localNode { get; set; } = new();
-        public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
-    }
-
-    public class CodexDebugTableNodeResponse
-    {
-        public string nodeId { get; set; } = string.Empty;
-        public string peerId { get; set; } = string.Empty;
-        public string record { get; set; } = string.Empty;
-        public string address { get; set; } = string.Empty;
-        public bool seen { get; set; }
-    }
-
-    public class EnginePeerResponse
-    {
-        public string peerId { get; set; } = string.Empty;
-        public EnginePeerContextResponse context { get; set; } = new();
-    }
-
-    public class EnginePeerContextResponse
-    {
-        public int blocks { get; set; } = 0;
-        public int peerWants { get; set; } = 0;
-        public int exchanged { get; set; } = 0;
-        public string lastExchange { get; set; } = string.Empty;
-    }
-
-    public class SwitchPeerResponse
-    {
-        public string peerId { get; set; } = string.Empty;
-        public string key { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugVersionResponse
-    {
-        public string version { get; set; } = string.Empty;
-        public string revision { get; set; } = string.Empty;
-
-        public bool IsValid()
-        {
-            return !string.IsNullOrEmpty(version) && !string.IsNullOrEmpty(revision);
-        }
-
-        public override string ToString()
-        {
-            return JsonConvert.SerializeObject(this);
-        }
-    }
-
-    public class CodexDebugPeerResponse
-    {
-        public bool IsPeerFound { get; set; }
-
-        public string peerId { get; set; } = string.Empty;
-        public long seqNo { get; set; }
-        public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
-    }
-
-    public class CodexDebugPeerAddressResponse
-    {
-        public string address { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugThresholdBreaches
-    {
-        public string[] breaches { get; set; } = Array.Empty<string>();
-    }
-
-    public class CodexSalesAvailabilityRequest
-    {
-        public string size { get; set; } = string.Empty;
-        public string duration { get; set; } = string.Empty;
-        public string minPrice { get; set; } = string.Empty;
-        public string maxCollateral { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesAvailabilityResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string size { get; set; } = string.Empty;
-        public string duration { get; set; } = string.Empty;
-        public string minPrice { get; set; } = string.Empty;
-        public string maxCollateral { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesRequestStorageRequest
-    {
-        public string duration { get; set; } = string.Empty;
-        public string proofProbability { get; set; } = string.Empty;
-        public string reward { get; set; } = string.Empty;
-        public string collateral { get; set; } = string.Empty;
-        public string? expiry { get; set; }
-        public uint? nodes { get; set; }
-        public uint? tolerance { get; set; }
-    }
-
-    public class CodexStoragePurchase
-    {
-        public string state { get; set; } = string.Empty;
-        public string error { get; set; } = string.Empty;
-    }
-}
@@ -1,96 +0,0 @@
-using DistTestCore.Marketplace;
-using KubernetesWorkflow;
-
-namespace DistTestCore.Codex
-{
-    public class CodexContainerRecipe : ContainerRecipeFactory
-    {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-7efa917";
-
-        public const string MetricsPortTag = "metrics_port";
-        public const string DiscoveryPortTag = "discovery-port";
-
-        // Used by tests for time-constraint assersions.
-        public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
-        public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
-
-        public override string AppName => "codex";
-        public override string Image { get; }
-
-        public CodexContainerRecipe()
-        {
-            Image = GetDockerImage();
-        }
-
-        protected override void Initialize(StartupConfig startupConfig)
-        {
-            var config = startupConfig.Get<CodexStartupConfig>();
-
-            AddExposedPortAndVar("CODEX_API_PORT");
-            AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
-
-            AddEnvVar("CODEX_DATA_DIR", $"datadir{ContainerNumber}");
-            AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
-            AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
-
-            // This makes the node announce itself to its local (pod) IP address.
-            AddEnvVar("NAT_IP_AUTO", "true");
-
-            var listenPort = AddInternalPort();
-            AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
-
-            if (!string.IsNullOrEmpty(config.BootstrapSpr))
-            {
-                AddEnvVar("CODEX_BOOTSTRAP_NODE", config.BootstrapSpr);
-            }
-            if (config.StorageQuota != null)
-            {
-                AddEnvVar("CODEX_STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
-            }
-            if (config.BlockTTL != null)
-            {
-                AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
-            }
-            if (config.MetricsEnabled)
-            {
-                AddEnvVar("CODEX_METRICS", "true");
-                AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
-                AddInternalPortAndVar("CODEX_METRICS_PORT", tag: MetricsPortTag);
-            }
-
-            if (config.MarketplaceConfig != null)
-            {
-                var gethConfig = startupConfig.Get<GethStartResult>();
-                var companionNode = gethConfig.CompanionNode;
-                var companionNodeAccount = companionNode.Accounts[GetAccountIndex(config.MarketplaceConfig)];
-                Additional(companionNodeAccount);
-
-                var ip = companionNode.RunningContainer.Pod.PodInfo.Ip;
-                var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number;
-
-                AddEnvVar("CODEX_ETH_PROVIDER", $"ws://{ip}:{port}");
-                AddEnvVar("CODEX_ETH_ACCOUNT", companionNodeAccount.Account);
-                AddEnvVar("CODEX_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
-                AddEnvVar("CODEX_PERSISTENCE", "true");
-
-                if (config.MarketplaceConfig.IsValidator)
-                {
-                    AddEnvVar("CODEX_VALIDATOR", "true");
-                }
-            }
-        }
-
-        private int GetAccountIndex(MarketplaceInitialConfig marketplaceConfig)
-        {
-            if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value;
-            return Index;
-        }
-
-        private string GetDockerImage()
-        {
-            var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
-            if (!string.IsNullOrEmpty(image)) return image;
-            return DefaultDockerImage;
-        }
-    }
-}
@@ -1,49 +0,0 @@
-using DistTestCore.Marketplace;
-using KubernetesWorkflow;
-
-namespace DistTestCore.Codex
-{
-    public class CodexDeployment
-    {
-        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata)
-        {
-            GethStartResult = gethStartResult;
-            CodexContainers = codexContainers;
-            PrometheusContainer = prometheusContainer;
-            Metadata = metadata;
-        }
-
-        public GethStartResult GethStartResult { get; }
-        public RunningContainer[] CodexContainers { get; }
-        public RunningContainer? PrometheusContainer { get; }
-        public DeploymentMetadata Metadata { get; }
-    }
-
-    public class DeploymentMetadata
-    {
-        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration)
-        {
-            DeployDateTimeUtc = DateTime.UtcNow;
-            KubeNamespace = kubeNamespace;
-            NumberOfCodexNodes = numberOfCodexNodes;
-            NumberOfValidators = numberOfValidators;
-            StorageQuotaMB = storageQuotaMB;
-            CodexLogLevel = codexLogLevel;
-            InitialTestTokens = initialTestTokens;
-            MinPrice = minPrice;
-            MaxCollateral = maxCollateral;
-            MaxDuration = maxDuration;
-        }
-
-        public DateTime DeployDateTimeUtc { get; }
-        public string KubeNamespace { get; }
-        public int NumberOfCodexNodes { get; }
-        public int NumberOfValidators { get; }
-        public int StorageQuotaMB { get; }
-        public CodexLogLevel CodexLogLevel { get; }
-        public int InitialTestTokens { get; }
-        public int MinPrice { get; }
-        public int MaxCollateral { get; }
-        public int MaxDuration { get; }
-    }
-}
@@ -1,22 +0,0 @@
-using DistTestCore.Marketplace;
-using KubernetesWorkflow;
-
-namespace DistTestCore.Codex
-{
-    public class CodexStartupConfig
-    {
-        public CodexStartupConfig(CodexLogLevel logLevel)
-        {
-            LogLevel = logLevel;
-        }
-
-        public string? NameOverride { get; set; }
-        public Location Location { get; set; }
-        public CodexLogLevel LogLevel { get; }
-        public ByteSize? StorageQuota { get; set; }
-        public bool MetricsEnabled { get; set; }
-        public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
-        public string? BootstrapSpr { get; set; }
-        public int? BlockTTL { get; set; }
-    }
-}
@@ -1,32 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Marketplace;
-using DistTestCore.Metrics;
-
-namespace DistTestCore
-{
-    public interface ICodexNodeFactory
-    {
-        OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group);
-    }
-
-    public class CodexNodeFactory : ICodexNodeFactory
-    {
-        private readonly TestLifecycle lifecycle;
-        private readonly IMetricsAccessFactory metricsAccessFactory;
-        private readonly IMarketplaceAccessFactory marketplaceAccessFactory;
-
-        public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory, IMarketplaceAccessFactory marketplaceAccessFactory)
-        {
-            this.lifecycle = lifecycle;
-            this.metricsAccessFactory = metricsAccessFactory;
-            this.marketplaceAccessFactory = marketplaceAccessFactory;
-        }
-
-        public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
-        {
-            var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container);
-            var marketplaceAccess = marketplaceAccessFactory.CreateMarketplaceAccess(access);
-            return new OnlineCodexNode(lifecycle, access, group, metricsAccess, marketplaceAccess);
-        }
-    }
-}
@@ -1,88 +0,0 @@
-using DistTestCore.Codex;
-using KubernetesWorkflow;
-using System.Collections;
-
-namespace DistTestCore
-{
-    public interface ICodexNodeGroup : IEnumerable<IOnlineCodexNode>
-    {
-        ICodexSetup BringOffline();
-        IOnlineCodexNode this[int index] { get; }
-    }
-
-    public class CodexNodeGroup : ICodexNodeGroup
-    {
-        private readonly TestLifecycle lifecycle;
-
-        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
-        {
-            this.lifecycle = lifecycle;
-            Setup = setup;
-            Containers = containers;
-            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
-            Version = new CodexDebugVersionResponse();
-        }
-
-        public IOnlineCodexNode this[int index]
-        {
-            get
-            {
-                return Nodes[index];
-            }
-        }
-
-        public ICodexSetup BringOffline()
-        {
-            lifecycle.CodexStarter.BringOffline(this);
-
-            var result = Setup;
-            // Clear everything. Prevent accidental use.
-            Setup = null!;
-            Nodes = Array.Empty<OnlineCodexNode>();
-            Containers = null!;
-
-            return result;
-        }
-
-        public CodexSetup Setup { get; private set; }
-        public RunningContainers[] Containers { get; private set; }
-        public OnlineCodexNode[] Nodes { get; private set; }
-        public CodexDebugVersionResponse Version { get; private set; }
-
-        public IEnumerator<IOnlineCodexNode> GetEnumerator()
-        {
-            return Nodes.Cast<IOnlineCodexNode>().GetEnumerator();
-        }
-
-        IEnumerator IEnumerable.GetEnumerator()
-        {
-            return Nodes.GetEnumerator();
-        }
-
-        public string Describe()
-        {
-            return $"group:[{Containers.Describe()}]";
-        }
-
-        public void EnsureOnline()
-        {
-            foreach (var node in Nodes) node.EnsureOnlineGetVersionResponse();
-            var versionResponses = Nodes.Select(n => n.Version);
-
-            var first = versionResponses.First();
-            if (!versionResponses.All(v => v.version == first.version && v.revision == first.revision))
-            {
-                throw new Exception("Inconsistent version information received from one or more Codex nodes: " +
-                    string.Join(",", versionResponses.Select(v => v.ToString())));
-            }
-
-            Version = first;
-        }
-
-        private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
-        {
-            var access = new CodexAccess(lifecycle.Log, c, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(c));
-            return factory.CreateOnlineCodexNode(access, this);
-        }
-    }
-}
@@ -1,95 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Marketplace;
-using KubernetesWorkflow;
-
-namespace DistTestCore
-{
-    public interface ICodexSetup
-    {
-        ICodexSetup WithName(string name);
-        ICodexSetup At(Location location);
-        ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
-        ICodexSetup WithStorageQuota(ByteSize storageQuota);
-        ICodexSetup WithBlockTTL(TimeSpan duration);
-        ICodexSetup EnableMetrics();
-        ICodexSetup EnableMarketplace(TestToken initialBalance);
-        ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
-        ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator);
-    }
-    
-    public class CodexSetup : CodexStartupConfig, ICodexSetup
-    {
-        public int NumberOfNodes { get; }
-
-        public CodexSetup(int numberOfNodes, CodexLogLevel logLevel)
-            : base(logLevel)
-        {
-            NumberOfNodes = numberOfNodes;
-        }
-
-        public ICodexSetup WithName(string name)
-        {
-            NameOverride = name;
-            return this;
-        }
-
-        public ICodexSetup At(Location location)
-        {
-            Location = location;
-            return this;
-        }
-
-        public ICodexSetup WithBootstrapNode(IOnlineCodexNode node)
-        {
-            BootstrapSpr = node.GetDebugInfo().spr;
-            return this;
-        }
-
-        public ICodexSetup WithStorageQuota(ByteSize storageQuota)
-        {
-            StorageQuota = storageQuota;
-            return this;
-        }
-
-        public ICodexSetup WithBlockTTL(TimeSpan duration)
-        {
-            BlockTTL = Convert.ToInt32(duration.TotalSeconds);
-            return this;
-        }
-
-        public ICodexSetup EnableMetrics()
-        {
-            MetricsEnabled = true;
-            return this;
-        }
-
-        public ICodexSetup EnableMarketplace(TestToken initialBalance)
-        {
-            return EnableMarketplace(initialBalance, 1000.Eth());
-        }
-
-        public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther)
-        {
-            return EnableMarketplace(initialBalance, initialEther, false);
-        }
-
-        public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator)
-        {
-            MarketplaceConfig = new MarketplaceInitialConfig(initialEther, initialBalance, isValidator);
-            return this;
-        }
-
-        public string Describe()
-        {
-            var args = string.Join(',', DescribeArgs());
-            return $"({NumberOfNodes} CodexNodes with args:[{args}])";
-        }
-
-        private IEnumerable<string> DescribeArgs()
-        {
-            yield return $"LogLevel={LogLevel}";
-            if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
-            if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
-        }
-    }
-}
@@ -1,132 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Marketplace;
-using DistTestCore.Metrics;
-using KubernetesWorkflow;
-using Logging;
-
-namespace DistTestCore
-{
-    public class CodexStarter : BaseStarter
-    {
-        public CodexStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
-        {
-        }
-
-        public List<CodexNodeGroup> RunningGroups { get; } = new List<CodexNodeGroup>();
-
-        public ICodexNodeGroup BringOnline(CodexSetup codexSetup)
-        {
-            LogSeparator();
-            LogStart($"Starting {codexSetup.Describe()}...");
-            var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
-
-            var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
-
-            var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
-
-            var metricAccessFactory = CollectMetrics(codexSetup, containers);
-            
-            var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
-
-            var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
-            lifecycle.SetCodexVersion(group.Version);
-
-            var nl = Environment.NewLine;
-            var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
-            LogEnd($"Started {codexSetup.NumberOfNodes} nodes " +
-                $"of image '{containers.Containers().First().Recipe.Image}' " +
-                $"and version '{group.Version}'{nl}" +
-                podInfos);
-            LogSeparator();
-
-            return group;
-        }
-
-        public void BringOffline(CodexNodeGroup group)
-        {
-            LogStart($"Stopping {group.Describe()}...");
-            var workflow = CreateWorkflow();
-            foreach (var c in group.Containers) workflow.Stop(c);
-            RunningGroups.Remove(group);
-            LogEnd("Stopped.");
-        }
-
-        public void DeleteAllResources()
-        {
-            var workflow = CreateWorkflow();
-            workflow.DeleteTestResources();
-
-            RunningGroups.Clear();
-        }
-
-        public void DownloadLog(RunningContainer container, ILogHandler logHandler)
-        {
-            var workflow = CreateWorkflow();
-            workflow.DownloadContainerLog(container, logHandler);
-        }
-
-        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
-        {
-            if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
-
-            var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
-            return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
-        }
-
-        private StartupConfig CreateStartupConfig(GethStartResult gethStartResult, CodexSetup codexSetup)
-        {
-            var startupConfig = new StartupConfig();
-            startupConfig.NameOverride = codexSetup.NameOverride;
-            startupConfig.Add(codexSetup);
-            startupConfig.Add(gethStartResult);
-            return startupConfig;
-        }
-
-        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
-        {
-            var result = new List<RunningContainers>();
-            var recipe = new CodexContainerRecipe();
-            for (var i = 0; i < numberOfNodes; i++)
-            {
-                var workflow = CreateWorkflow();
-                result.Add(workflow.Start(1, location, recipe, startupConfig));
-            }
-            return result.ToArray();
-        }
-
-        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
-        {
-            var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
-            RunningGroups.Add(group);
-
-            try
-            {
-                Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
-            }
-            catch
-            {
-                CodexNodesNotOnline(runningContainers);
-                throw;
-            }
-
-            return group;
-        }
-
-        private void CodexNodesNotOnline(RunningContainers[] runningContainers)
-        {
-            Log("Codex nodes failed to start");
-            foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
-        }
-
-        private StartupWorkflow CreateWorkflow()
-        {
-            return lifecycle.WorkflowCreator.CreateWorkflow();
-        }
-
-        private void LogSeparator()
-        {
-            Log("----------------------------------------------------------------------------");
-        }
-    }
-}
@@ -1,144 +0,0 @@
-using DistTestCore.Codex;
-using KubernetesWorkflow;
-using System.Net.NetworkInformation;
-using Utils;
-
-namespace DistTestCore
-{
-    public class Configuration
-    {
-        private readonly string? kubeConfigFile;
-        private readonly string logPath;
-        private readonly bool logDebug;
-        private readonly string dataFilesPath;
-        private readonly CodexLogLevel codexLogLevel;
-        private readonly string k8sNamespacePrefix;
-        private RunnerLocation? runnerLocation = null;
-
-        public Configuration()
-        {
-            kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
-            logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
-            logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
-            dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
-            codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
-            k8sNamespacePrefix = "ct-";
-        }
-
-        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, string k8sNamespacePrefix)
-        {
-            this.kubeConfigFile = kubeConfigFile;
-            this.logPath = logPath;
-            this.logDebug = logDebug;
-            this.dataFilesPath = dataFilesPath;
-            this.codexLogLevel = codexLogLevel;
-            this.k8sNamespacePrefix = k8sNamespacePrefix;
-        }
-
-        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
-        {
-            return new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: k8sNamespacePrefix,
-                kubeConfigFile: kubeConfigFile,
-                operationTimeout: timeSet.K8sOperationTimeout(),
-                retryDelay: timeSet.WaitForK8sServiceDelay()
-            );
-        }
-
-        public Logging.LogConfig GetLogConfig()
-        {
-            return new Logging.LogConfig(logPath, debugEnabled: logDebug);
-        }
-
-        public string GetFileManagerFolder()
-        {
-            return dataFilesPath;
-        }
-
-        public CodexLogLevel GetCodexLogLevel()
-        {
-            return codexLogLevel;
-        }
-
-        public Address GetAddress(RunningContainer container)
-        {
-            if (runnerLocation == null)
-            {
-                runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container);
-            }
-            
-            if (runnerLocation == RunnerLocation.InternalToCluster)
-            {
-                return container.ClusterInternalAddress;
-            }
-            return container.ClusterExternalAddress;
-        }
-
-        private static string GetEnvVarOrDefault(string varName, string defaultValue)
-        {
-            var v = Environment.GetEnvironmentVariable(varName);
-            if (v == null) return defaultValue;
-            return v;
-        }
-
-        private static string? GetNullableEnvVarOrDefault(string varName, string? defaultValue)
-        {
-            var v = Environment.GetEnvironmentVariable(varName);
-            if (v == null) return defaultValue;
-            return v;
-        }
-    }
-
-    public enum RunnerLocation
-    {
-        ExternalToCluster,
-        InternalToCluster,
-    }
-
-    public static class RunnerLocationUtils
-    {
-        private static bool alreadyDidThat = false;
-
-        public static RunnerLocation DetermineRunnerLocation(RunningContainer container)
-        {
-            // We want to be sure we don't ping more often than strictly necessary.
-            // If we have already determined the location during this application
-            // lifetime, don't do it again.
-            if (alreadyDidThat) throw new Exception("We already did that.");
-            alreadyDidThat = true;
-
-            if (PingHost(container.Pod.PodInfo.Ip))
-            {
-                return RunnerLocation.InternalToCluster;
-            }
-            if (PingHost(Format(container.ClusterExternalAddress)))
-            {
-                return RunnerLocation.ExternalToCluster;
-            }
-
-            throw new Exception("Unable to determine runner location.");
-        }
-
-        private static string Format(Address host)
-        {
-            return host.Host
-                .Replace("http://", "")
-                .Replace("https://", "");
-        }
-
-        private static bool PingHost(string host)
-        {
-            try
-            {
-                using var pinger = new Ping();
-                PingReply reply = pinger.Send(host);
-                return reply.Status == IPStatus.Success;
-            }
-            catch (PingException)
-            {
-            }
-
-            return false;
-        }
-    }
-}
@@ -1,315 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Helpers;
-using DistTestCore.Logs;
-using DistTestCore.Marketplace;
-using DistTestCore.Metrics;
-using KubernetesWorkflow;
-using Logging;
-using NUnit.Framework;
-using System.Reflection;
-
-namespace DistTestCore
-{
-    [Parallelizable(ParallelScope.All)]
-    public abstract class DistTest
-    {
-        private const string TestsType = "dist-tests";
-        private readonly Configuration configuration = new Configuration();
-        private readonly Assembly[] testAssemblies;
-        private readonly FixtureLog fixtureLog;
-        private readonly StatusLog statusLog;
-        private readonly object lifecycleLock = new object();
-        private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
-
-        public DistTest()
-        {
-            var assemblies = AppDomain.CurrentDomain.GetAssemblies();
-            testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
-
-            var logConfig = configuration.GetLogConfig();
-            var startTime = DateTime.UtcNow;
-            fixtureLog = new FixtureLog(logConfig, startTime);
-            statusLog = new StatusLog(logConfig, startTime);
-
-            PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
-            PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
-        }
-
-        public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; }
-        public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; }
-
-        [OneTimeSetUp]
-        public void GlobalSetup()
-        {
-            fixtureLog.Log($"Codex Distributed Tests are starting...");
-            fixtureLog.Log($"Codex image: '{new CodexContainerRecipe().Image}'");
-            fixtureLog.Log($"CodexContracts image: '{new CodexContractsContainerRecipe().Image}'");
-            fixtureLog.Log($"Prometheus image: '{new PrometheusContainerRecipe().Image}'");
-            fixtureLog.Log($"Geth image: '{new GethContainerRecipe().Image}'");
-
-            // Previous test run may have been interrupted.
-            // Begin by cleaning everything up.
-            try
-            {
-                Stopwatch.Measure(fixtureLog, "Global setup", () =>
-                {
-                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), new PodLabels(TestsType, null!), string.Empty);
-                    wc.CreateWorkflow().DeleteAllResources();
-                });                
-            }
-            catch (Exception ex)
-            {
-                GlobalTestFailure.HasFailed = true;
-                fixtureLog.Error($"Global setup cleanup failed with: {ex}");
-                throw;
-            }
-
-            fixtureLog.Log("Global setup cleanup successful");
-        }
-
-        [SetUp]
-        public void SetUpDistTest()
-        {
-            if (GlobalTestFailure.HasFailed)
-            {
-                Assert.Inconclusive("Skip test: Previous test failed during clean up.");
-            }
-            else
-            {
-                CreateNewTestLifecycle();
-            }
-        }
-
-        [TearDown]
-        public void TearDownDistTest()
-        {
-            try
-            {
-                DisposeTestLifecycle();
-            }
-            catch (Exception ex)
-            {
-                fixtureLog.Error("Cleanup failed: " + ex.Message);
-                GlobalTestFailure.HasFailed = true;
-            }
-        }
-
-        public TestFile GenerateTestFile(ByteSize size, string label = "")
-        {
-            return Get().FileManager.GenerateTestFile(size, label);
-        }
-
-        /// <summary>
-        /// Any test files generated in 'action' will be deleted after it returns.
-        /// This helps prevent large tests from filling up discs.
-        /// </summary>
-        public void ScopedTestFiles(Action action)
-        {
-            Get().FileManager.PushFileSet();
-            action();
-            Get().FileManager.PopFileSet();
-        }
-
-        public IOnlineCodexNode SetupCodexBootstrapNode()
-        {
-            return SetupCodexBootstrapNode(s => { });
-        }
-
-        public virtual IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
-        {
-            return SetupCodexNode(s =>
-            {
-                setup(s);
-                s.WithName("Bootstrap");
-            });
-        }
-
-        public IOnlineCodexNode SetupCodexNode()
-        {
-            return SetupCodexNode(s => { });
-        }
-
-        public IOnlineCodexNode SetupCodexNode(Action<ICodexSetup> setup)
-        {
-            return SetupCodexNodes(1, setup)[0];
-        }
-
-        public ICodexNodeGroup SetupCodexNodes(int numberOfNodes)
-        {
-            return SetupCodexNodes(numberOfNodes, s => { });
-        }
-
-        public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
-        {
-            var codexSetup = CreateCodexSetup(numberOfNodes);
-
-            setup(codexSetup);
-
-            return BringOnline(codexSetup);
-        }
-
-        public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
-        {
-            return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
-        }
-
-        public IEnumerable<IOnlineCodexNode> GetAllOnlineCodexNodes()
-        {
-            return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
-        }
-
-        public BaseLog GetTestLog()
-        {
-            return Get().Log;
-        }
-
-        public void Log(string msg)
-        {
-            TestContext.Progress.WriteLine(msg);
-            GetTestLog().Log(msg);
-        }
-
-        public void Debug(string msg)
-        {
-            TestContext.Progress.WriteLine(msg);
-            GetTestLog().Debug(msg);
-        }
-
-        protected CodexSetup CreateCodexSetup(int numberOfNodes)
-        {
-            return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
-        }
-
-        private TestLifecycle Get()
-        {
-            lock (lifecycleLock)
-            {
-                return lifecycles[GetCurrentTestName()];
-            }
-        }
-
-        private void CreateNewTestLifecycle()
-        {
-            var testName = GetCurrentTestName();
-            fixtureLog.WriteLogTag();
-            Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
-            {
-                lock (lifecycleLock)
-                {
-                    var testNamespace = Guid.NewGuid().ToString();
-                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), TestsType, testNamespace);
-                    lifecycles.Add(testName, lifecycle);
-                }
-            });
-        }
-
-        private void DisposeTestLifecycle()
-        {
-            var lifecycle = Get();
-            var testResult = GetTestResult();
-            var testDuration = lifecycle.GetTestDuration();
-            fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
-            statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetApplicationIds());
-            Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
-            {
-                lifecycle.Log.EndTest();
-                IncludeLogsAndMetricsOnTestFailure(lifecycle);
-                lifecycle.DeleteAllResources();
-                lifecycle = null!;
-            });
-        }
-
-        private ITimeSet GetTimeSet()
-        {
-            if (ShouldUseLongTimeouts()) return new LongTimeSet();
-            return new DefaultTimeSet();
-        }
-
-        private bool ShouldUseLongTimeouts()
-        {
-            // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
-            // But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
-            // if the attribute is present.
-            var currentTest = TestContext.CurrentContext.Test;
-            var className = currentTest.ClassName;
-            var methodName = currentTest.MethodName;
-
-            var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
-            var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
-
-            return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
-        }
-
-        private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
-        {
-            var result = TestContext.CurrentContext.Result;
-            if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
-            {
-                fixtureLog.MarkAsFailed();
-
-                if (IsDownloadingLogsAndMetricsEnabled())
-                {
-                    lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
-                    DownloadAllLogs(lifecycle);
-                    DownloadAllMetrics(lifecycle);
-                }
-                else
-                {
-                    lifecycle.Log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
-                }
-            }
-        }
-
-        private void DownloadAllLogs(TestLifecycle lifecycle)
-        {
-            OnEachCodexNode(lifecycle, node =>
-            {
-                lifecycle.DownloadLog(node.CodexAccess.Container);
-            });
-        }
-
-        private void DownloadAllMetrics(TestLifecycle lifecycle)
-        {
-            var metricsDownloader = new MetricsDownloader(lifecycle.Log);
-
-            OnEachCodexNode(lifecycle, node =>
-            {
-                var m = node.Metrics as MetricsAccess;
-                if (m != null)
-                {
-                    metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m);
-                }
-            });
-        }
-
-        private void OnEachCodexNode(TestLifecycle lifecycle, Action<OnlineCodexNode> action)
-        {
-            var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
-            foreach (var node in allNodes)
-            {
-                action(node);
-            }
-        }
-
-        private string GetCurrentTestName()
-        {
-            return $"[{TestContext.CurrentContext.Test.Name}]";
-        }
-
-        private string GetTestResult()
-        {
-            return TestContext.CurrentContext.Result.Outcome.Status.ToString();
-        }
-
-        private bool IsDownloadingLogsAndMetricsEnabled()
-        {
-            var testProperties = TestContext.CurrentContext.Test.Properties;
-            return !testProperties.ContainsKey(DontDownloadLogsAndMetricsOnFailureAttribute.DontDownloadKey);
-        }
-    }
-
-    public static class GlobalTestFailure
-    {
-        public static bool HasFailed { get; set; } = false;
-    }
-}
@@ -1,218 +0,0 @@
-using Logging;
-using NUnit.Framework;
-using Utils;
-
-namespace DistTestCore
-{
-    public interface IFileManager
-    {
-        TestFile CreateEmptyTestFile(string label = "");
-        TestFile GenerateTestFile(ByteSize size, string label = "");
-        void DeleteAllTestFiles();
-        void PushFileSet();
-        void PopFileSet();
-    }
-
-    public class FileManager : IFileManager
-    {
-        public const int ChunkSize = 1024 * 1024 * 100;
-        private static NumberSource folderNumberSource = new NumberSource(0);
-        private readonly Random random = new Random();
-        private readonly BaseLog log;
-        private readonly string folder;
-        private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();
-
-        public FileManager(BaseLog log, Configuration configuration)
-        {
-            folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5"));
-
-            EnsureDirectory();
-            this.log = log;
-        }
-
-        public TestFile CreateEmptyTestFile(string label = "")
-        {
-            var path = Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin");
-            var result = new TestFile(log, path, label);
-            File.Create(result.Filename).Close();
-            if (fileSetStack.Any()) fileSetStack.Last().Add(result);
-            return result;
-        }
-
-        public TestFile GenerateTestFile(ByteSize size, string label)
-        {
-            var sw = Stopwatch.Begin(log);
-            var result = GenerateFile(size, label);
-            sw.End($"Generated file '{result.Describe()}'.");
-            return result;
-        }
-
-        public void DeleteAllTestFiles()
-        {
-            DeleteDirectory();
-        }
-
-        public void PushFileSet()
-        {
-            fileSetStack.Add(new List<TestFile>());
-        }
-
-        public void PopFileSet()
-        {
-            if (!fileSetStack.Any()) return;
-            var pop = fileSetStack.Last();
-            fileSetStack.Remove(pop);
-
-            foreach (var file in pop)
-            {
-                try
-                {
-                    File.Delete(file.Filename);
-                }
-                catch { }
-            }
-        }
-
-        private TestFile GenerateFile(ByteSize size, string label)
-        {
-            var result = CreateEmptyTestFile(label);
-            CheckSpaceAvailable(result, size);
-
-            GenerateFileBytes(result, size);
-            return result;
-        }
-
-        private void CheckSpaceAvailable(TestFile testFile, ByteSize size)
-        {
-            var file = new FileInfo(testFile.Filename);
-            var drive = new DriveInfo(file.Directory!.Root.FullName);
-
-            var spaceAvailable = drive.TotalFreeSpace;
-
-            if (spaceAvailable < size.SizeInBytes)
-            {
-                var msg = $"Inconclusive: Not enough disk space to perform test. " +
-                    $"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
-                    $"{Formatter.FormatByteSize(spaceAvailable)} available.";
-
-                log.Log(msg);
-                Assert.Inconclusive(msg);
-            }
-        }
-
-        private void GenerateFileBytes(TestFile result, ByteSize size)
-        {
-            long bytesLeft = size.SizeInBytes;
-            int chunkSize = ChunkSize;
-            while (bytesLeft > 0)
-            {
-                try
-                {
-                    var length = Math.Min(bytesLeft, chunkSize);
-                    AppendRandomBytesToFile(result, length);
-                    bytesLeft -= length;
-                }
-                catch
-                {
-                    chunkSize = chunkSize / 2;
-                    if (chunkSize < 1024) throw;
-                }
-            }
-        }
-
-        private void AppendRandomBytesToFile(TestFile result, long length)
-        {
-            var bytes = new byte[length];
-            random.NextBytes(bytes);
-            using var stream = new FileStream(result.Filename, FileMode.Append);
-            stream.Write(bytes, 0, bytes.Length);
-        }
-
-        private void EnsureDirectory()
-        {
-            if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
-        }
-
-        private void DeleteDirectory()
-        {
-            Directory.Delete(folder, true);
-        }
-    }
-
-    public class TestFile
-    {
-        private readonly BaseLog log;
-
-        public TestFile(BaseLog log, string filename, string label)
-        {
-            this.log = log;
-            Filename = filename;
-            Label = label;
-        }
-
-        public string Filename { get; }
-        public string Label { get; }
-
-        public void AssertIsEqual(TestFile? actual)
-        {
-            var sw = Stopwatch.Begin(log);
-            try
-            {
-                AssertEqual(actual);
-            }
-            finally
-            {
-                sw.End($"{nameof(TestFile)}.{nameof(AssertIsEqual)}");
-            }
-        }
-
-        public string Describe()
-        {
-            var sizePostfix = $" ({Formatter.FormatByteSize(GetFileSize())})";
-            if (!string.IsNullOrEmpty(Label)) return Label + sizePostfix;
-            return $"'{Filename}'{sizePostfix}";
-        }
-
-        private void AssertEqual(TestFile? actual)
-        {
-            if (actual == null) Assert.Fail("TestFile is null.");
-            if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
-
-            Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length.");
-
-            using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
-            using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
-
-            var bytesExpected = new byte[FileManager.ChunkSize];
-            var bytesActual = new byte[FileManager.ChunkSize];
-
-            var readExpected = 0;
-            var readActual = 0;
-
-            while (true)
-            {
-                readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
-                readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
-
-                if (readExpected == 0 && readActual == 0)
-                {
-                    log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
-                    return;
-                }
-
-                Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length.");
-
-                for (var i = 0; i < readActual; i++)
-                {
-                    if (bytesExpected[i] != bytesActual[i]) Assert.Fail("File contents not equal.");
-                }
-            }
-        }
-
-        private long GetFileSize()
-        {
-            var info = new FileInfo(Filename);
-            return info.Length;
-        }
-    }
-}
@@ -1,88 +0,0 @@
-using DistTestCore.Marketplace;
-
-namespace DistTestCore
-{
-    public class GethStarter : BaseStarter
-    {
-        private readonly MarketplaceNetworkCache marketplaceNetworkCache;
-        private readonly GethCompanionNodeStarter companionNodeStarter;
-
-        public GethStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
-        {
-            marketplaceNetworkCache = new MarketplaceNetworkCache(
-                new GethBootstrapNodeStarter(lifecycle),
-                new CodexContractsStarter(lifecycle));
-            companionNodeStarter = new GethCompanionNodeStarter(lifecycle);
-        }
-
-        public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup)
-        {
-            if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult();
-
-            var marketplaceNetwork = marketplaceNetworkCache.Get();
-            var companionNode = StartCompanionNode(codexSetup, marketplaceNetwork);
-
-            LogStart("Setting up initial balance...");
-            TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNode);
-            LogEnd($"Initial balance of {codexSetup.MarketplaceConfig.InitialTestTokens} set for {codexSetup.NumberOfNodes} nodes.");
-
-            return CreateGethStartResult(marketplaceNetwork, companionNode);
-        }
-
-        private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
-        {
-            if (marketplaceConfig.InitialTestTokens.Amount == 0) return;
-
-            var interaction = marketplaceNetwork.StartInteraction(lifecycle);
-            var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
-
-            var accounts = companionNode.Accounts.Select(a => a.Account).ToArray();
-            interaction.MintTestTokens(accounts, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
-        }
-
-        private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
-        {
-            return new GethStartResult(CreateMarketplaceAccessFactory(marketplaceNetwork), marketplaceNetwork, companionNode);
-        }
-
-        private GethStartResult CreateMarketplaceUnavailableResult()
-        {
-            return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, null!);
-        }
-
-        private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork)
-        {
-            return new GethMarketplaceAccessFactory(lifecycle, marketplaceNetwork);
-        }
-
-        private GethCompanionNodeInfo StartCompanionNode(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)
-        {
-            return companionNodeStarter.StartCompanionNodeFor(codexSetup, marketplaceNetwork);
-        }
-    }
-
-    public class MarketplaceNetworkCache
-    {
-        private readonly GethBootstrapNodeStarter bootstrapNodeStarter;
-        private readonly CodexContractsStarter codexContractsStarter;
-        private MarketplaceNetwork? network;
-
-        public MarketplaceNetworkCache(GethBootstrapNodeStarter bootstrapNodeStarter, CodexContractsStarter codexContractsStarter)
-        {
-            this.bootstrapNodeStarter = bootstrapNodeStarter;
-            this.codexContractsStarter = codexContractsStarter;
-        }
-
-        public MarketplaceNetwork Get()
-        {
-            if (network == null)
-            {
-                var bootstrapInfo = bootstrapNodeStarter.StartGethBootstrapNode();
-                var marketplaceInfo = codexContractsStarter.Start(bootstrapInfo);
-                network = new MarketplaceNetwork(bootstrapInfo, marketplaceInfo );
-            }
-            return network;
-        }
-    }
-}
@@ -1,138 +0,0 @@
-using Logging;
-using Newtonsoft.Json;
-using System.Net.Http.Headers;
-using System.Net.Http.Json;
-using Utils;
-
-namespace DistTestCore
-{
-    public class Http
-    {
-        private readonly BaseLog log;
-        private readonly ITimeSet timeSet;
-        private readonly Address address;
-        private readonly string baseUrl;
-        private readonly string? logAlias;
-
-        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
-        {
-            this.log = log;
-            this.timeSet = timeSet;
-            this.address = address;
-            this.baseUrl = baseUrl;
-            this.logAlias = logAlias;
-            if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
-            if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
-        }
-
-        public string HttpGetString(string route)
-        {
-            return Retry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, "");
-                var result = Time.Wait(client.GetAsync(url));
-                var str = Time.Wait(result.Content.ReadAsStringAsync());
-                Log(url, str);
-                return str; ;
-            }, $"HTTP-GET:{route}");
-        }
-
-        public T HttpGetJson<T>(string route)
-        {
-            var json = HttpGetString(route);
-            return TryJsonDeserialize<T>(json);
-        }
-
-        public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
-        {
-            var json = HttpPostJson(route, body);
-            return TryJsonDeserialize<TResponse>(json);
-        }
-
-        public string HttpPostJson<TRequest>(string route, TRequest body)
-        {
-            return Retry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                using var content = JsonContent.Create(body);
-                Log(url, JsonConvert.SerializeObject(body));
-                var result = Time.Wait(client.PostAsync(url, content));
-                var str = Time.Wait(result.Content.ReadAsStringAsync());
-                Log(url, str);
-                return str;
-            }, $"HTTP-POST-JSON: {route}");
-        }
-
-        public string HttpPostStream(string route, Stream stream)
-        {
-            return Retry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, "~ STREAM ~");
-                var content = new StreamContent(stream);
-                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
-                var response = Time.Wait(client.PostAsync(url, content));
-                var str =Time.Wait(response.Content.ReadAsStringAsync());
-                Log(url, str);
-                return str;
-            }, $"HTTP-POST-STREAM: {route}");
-        }
-
-        public Stream HttpGetStream(string route)
-        {
-            return Retry(() =>
-            {
-                var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, "~ STREAM ~");
-                return Time.Wait(client.GetStreamAsync(url));
-            }, $"HTTP-GET-STREAM: {route}");
-        }
-
-        public T TryJsonDeserialize<T>(string json)
-        {
-            try
-            {
-                return JsonConvert.DeserializeObject<T>(json)!;
-            }
-            catch (Exception exception)
-            {
-                var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}";
-                throw new InvalidOperationException(msg, exception);
-            }
-        }
-
-        private string GetUrl()
-        {
-            return $"{address.Host}:{address.Port}{baseUrl}";
-        }
-
-        private void Log(string url, string message)
-        {
-            if (logAlias != null)
-            {
-                log.Debug($"({logAlias})({url}) = '{message}'", 3);
-            }
-            else
-            {
-                log.Debug($"({url}) = '{message}'", 3);
-            }
-        }
-
-        private T Retry<T>(Func<T> operation, string description)
-        {
-            return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
-        }
-
-        private HttpClient GetClient()
-        {
-            var client = new HttpClient();
-            client.Timeout = timeSet.HttpCallTimeout();
-            return client;
-        }
-    }
-}
@@ -1,15 +0,0 @@
-using NUnit.Framework;
-
-namespace DistTestCore.Logs
-{
-    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
-    public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute
-    {
-        public const string DontDownloadKey = "DontDownloadLogsAndMetrics";
-
-        public DontDownloadLogsAndMetricsOnFailureAttribute()
-            : base(DontDownloadKey)
-        {
-        }
-    }
-}
@@ -1,37 +0,0 @@
-using Logging;
-using NUnit.Framework;
-
-namespace DistTestCore.Logs
-{
-    public interface IDownloadedLog
-    {
-        void AssertLogContains(string expectedString);
-    }
-
-    public class DownloadedLog : IDownloadedLog
-    {
-        private readonly LogFile logFile;
-        private readonly string owner;
-
-        public DownloadedLog(LogFile logFile, string owner)
-        {
-            this.logFile = logFile;
-            this.owner = owner;
-        }
-
-        public void AssertLogContains(string expectedString)
-        {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (line.Contains(expectedString)) return;
-                line = streamReader.ReadLine();
-            }
-
-            Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
-        }
-    }
-}
@@ -1,30 +0,0 @@
-using KubernetesWorkflow;
-using Logging;
-
-namespace DistTestCore.Logs
-{
-    public class LogDownloadHandler : LogHandler, ILogHandler
-    {
-        private readonly RunningContainer container;
-        private readonly LogFile log;
-
-        public LogDownloadHandler(RunningContainer container, string description, LogFile log)
-        {
-            this.container = container;
-            this.log = log;
-
-            log.Write($"{description} -->> {log.FullFilename}");
-            log.WriteRaw(description);
-        }
-
-        public DownloadedLog DownloadLog()
-        {
-            return new DownloadedLog(log, container.Name);
-        }
-
-        protected override void ProcessLine(string line)
-        {
-            log.WriteRaw(line);
-        }
-    }
-}
@@ -1,16 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore.Marketplace
-{
-    public class CodexContractsContainerConfig
-    {
-        public CodexContractsContainerConfig(string bootstrapNodeIp, Port jsonRpcPort)
-        {
-            BootstrapNodeIp = bootstrapNodeIp;
-            JsonRpcPort = jsonRpcPort;
-        }
-
-        public string BootstrapNodeIp { get; }
-        public Port JsonRpcPort { get; }
-    }
-}
@@ -1,103 +0,0 @@
-using KubernetesWorkflow;
-using Utils;
-
-namespace DistTestCore.Marketplace
-{
-    public class CodexContractsStarter : BaseStarter
-    {
-
-        public CodexContractsStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
-        {
-        }
-
-        public MarketplaceInfo Start(GethBootstrapNodeInfo bootstrapNode)
-        {
-            LogStart("Deploying Codex Marketplace...");
-
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
-            var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]);
-
-            var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig);
-            if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure.");
-            var container = containers.Containers[0];
-
-            WaitUntil(() =>
-            {
-                var logHandler = new ContractsReadyLogHandler(Debug);
-                workflow.DownloadContainerLog(container, logHandler);
-                return logHandler.Found;
-            });
-            Log("Contracts deployed. Extracting addresses...");
-
-            var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
-            var marketplaceAddress = extractor.ExtractMarketplaceAddress();
-            var abi = extractor.ExtractMarketplaceAbi();
-
-            var interaction = bootstrapNode.StartInteraction(lifecycle);
-            var tokenAddress = interaction.GetTokenAddress(marketplaceAddress);
-
-            LogEnd("Extract completed. Marketplace deployed.");
-
-            return new MarketplaceInfo(marketplaceAddress, abi, tokenAddress);
-        }
-
-        private void WaitUntil(Func<bool> predicate)
-        {
-            Time.WaitUntil(predicate, TimeSpan.FromMinutes(3), TimeSpan.FromSeconds(2));
-        }
-
-        private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer)
-        {
-            var startupConfig = new StartupConfig();
-            var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.PodInfo.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
-            startupConfig.Add(contractsConfig);
-            return startupConfig;
-        }
-    }
-
-    public class MarketplaceInfo
-    {
-        public MarketplaceInfo(string address, string abi, string tokenAddress)
-        {
-            Address = address;
-            Abi = abi;
-            TokenAddress = tokenAddress;
-        }
-
-        public string Address { get; }
-        public string Abi { get; }
-        public string TokenAddress { get; }
-    }
-
-    public class ContractsReadyLogHandler : LogHandler
-    {
-        // Log should contain 'Compiled 15 Solidity files successfully' at some point.
-        private const string RequiredCompiledString = "Solidity files successfully";
-        // When script is done, it prints the ready-string.
-        private const string ReadyString = "Done! Sleeping indefinitely...";
-        private readonly Action<string> debug;
-
-        public ContractsReadyLogHandler(Action<string> debug)
-        {
-            this.debug = debug;
-            debug($"Looking for '{RequiredCompiledString}' and '{ReadyString}' in container logs...");
-        }
-
-        public bool SeenCompileString { get; private set; }
-        public bool Found { get; private set; }
-
-        protected override void ProcessLine(string line)
-        {
-            debug(line);
-            if (line.Contains(RequiredCompiledString)) SeenCompileString = true;
-            if (line.Contains(ReadyString))
-            {
-                if (!SeenCompileString) throw new Exception("CodexContracts deployment failed. " +
-                    "Solidity files not compiled before process exited.");
-
-                Found = true;
-            }
-        }
-    }
-}
@@ -1,42 +0,0 @@
-using KubernetesWorkflow;
-using NethereumWorkflow;
-
-namespace DistTestCore.Marketplace
-{
-    public class GethBootstrapNodeInfo
-    {
-        public GethBootstrapNodeInfo(RunningContainers runningContainers, AllGethAccounts allAccounts, string pubKey, Port discoveryPort)
-        {
-            RunningContainers = runningContainers;
-            AllAccounts = allAccounts;
-            Account = allAccounts.Accounts[0];
-            PubKey = pubKey;
-            DiscoveryPort = discoveryPort;
-        }
-
-        public RunningContainers RunningContainers { get; }
-        public AllGethAccounts AllAccounts { get; }
-        public GethAccount Account { get; }
-        public string PubKey { get; }
-        public Port DiscoveryPort { get; }
-
-        public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
-        {
-            var address = lifecycle.Configuration.GetAddress(RunningContainers.Containers[0]);
-            var account = Account;
-
-            var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, account.PrivateKey);
-            return creator.CreateWorkflow();
-        }
-    }
-
-    public class AllGethAccounts
-    {
-        public GethAccount[] Accounts { get; }
-
-        public AllGethAccounts(GethAccount[] accounts)
-        {
-            Accounts = accounts;
-        }
-    }
-}
@@ -1,40 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore.Marketplace
-{
-    public class GethBootstrapNodeStarter : BaseStarter
-    {
-        public GethBootstrapNodeStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
-        {
-        }
-
-        public GethBootstrapNodeInfo StartGethBootstrapNode()
-        {
-            LogStart("Starting Geth bootstrap node...");
-            var startupConfig = CreateBootstrapStartupConfig();
-            
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
-            var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
-            if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
-            var bootstrapContainer = containers.Containers[0];
-
-            var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer);
-            var accounts = extractor.ExtractAccounts();
-            var pubKey = extractor.ExtractPubKey();
-            var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
-            var result = new GethBootstrapNodeInfo(containers, accounts, pubKey, discoveryPort);
-
-            LogEnd($"Geth bootstrap node started with account '{result.Account.Account}'");
-
-            return result;
-        }
-
-        private StartupConfig CreateBootstrapStartupConfig()
-        {
-            var config = new StartupConfig();
-            config.Add(new GethStartupConfig(true, null!, 0, 0));
-            return config;
-        }
-    }
-}
@@ -1,38 +0,0 @@
-using KubernetesWorkflow;
-using NethereumWorkflow;
-
-namespace DistTestCore.Marketplace
-{
-    public class GethCompanionNodeInfo
-    {
-        public GethCompanionNodeInfo(RunningContainer runningContainer, GethAccount[] accounts)
-        {
-            RunningContainer = runningContainer;
-            Accounts = accounts;
-        }
-
-        public RunningContainer RunningContainer { get; }
-        public GethAccount[] Accounts { get; }
-        
-        public NethereumInteraction StartInteraction(TestLifecycle lifecycle, GethAccount account)
-        {
-            var address = lifecycle.Configuration.GetAddress(RunningContainer);
-            var privateKey = account.PrivateKey;
-
-            var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, privateKey);
-            return creator.CreateWorkflow();
-        }
-    }
-
-    public class GethAccount
-    {
-        public GethAccount(string account, string privateKey)
-        {
-            Account = account;
-            PrivateKey = privateKey;
-        }
-
-        public string Account { get; }
-        public string PrivateKey { get; }
-    }
-}
@@ -1,77 +0,0 @@
-using KubernetesWorkflow;
-using Utils;
-
-namespace DistTestCore.Marketplace
-{
-    public class GethCompanionNodeStarter : BaseStarter
-    {
-        private int companionAccountIndex = 0;
-
-        public GethCompanionNodeStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
-        {
-        }
-
-        public GethCompanionNodeInfo StartCompanionNodeFor(CodexSetup codexSetup, MarketplaceNetwork marketplace)
-        {
-            LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes.");
-
-            var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
-
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
-            var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
-            if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
-            var container = containers.Containers[0];
-
-            var node = CreateCompanionInfo(container, marketplace, config);
-            EnsureCompanionNodeIsSynced(node, marketplace);
-
-            LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]");
-            return node;
-        }
-
-        private GethCompanionNodeInfo CreateCompanionInfo(RunningContainer container, MarketplaceNetwork marketplace, GethStartupConfig config)
-        {
-            var accounts = ExtractAccounts(marketplace, config);
-            return new GethCompanionNodeInfo(container, accounts);
-        }
-
-        private static GethAccount[] ExtractAccounts(MarketplaceNetwork marketplace, GethStartupConfig config)
-        {
-            return marketplace.Bootstrap.AllAccounts.Accounts
-                .Skip(1 + config.CompanionAccountStartIndex)
-                .Take(config.NumberOfCompanionAccounts)
-                .ToArray();
-        }
-
-        private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
-        {
-            try
-            {
-                Time.WaitUntil(() =>
-                {
-                    var interaction = node.StartInteraction(lifecycle, node.Accounts.First());
-                    return interaction.IsSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
-                }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
-            }
-            catch (Exception e)
-            {
-                throw new Exception("Geth companion node did not sync within timeout. Test infra failure.", e);
-            }
-        }
-
-        private GethStartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
-        {
-            var config = new GethStartupConfig(false, bootstrapNode, companionAccountIndex, numberOfAccounts);
-            companionAccountIndex += numberOfAccounts;
-            return config;
-        }
-
-        private StartupConfig CreateStartupConfig(GethStartupConfig gethConfig)
-        {
-            var config = new StartupConfig();
-            config.Add(gethConfig);
-            return config;
-        }
-    }
-}
@@ -1,73 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore.Marketplace
-{
-    public class GethContainerRecipe : ContainerRecipeFactory
-    {
-        private const string defaultArgs = "--ipcdisable --syncmode full";
-
-        public const string HttpPortTag = "http_port";
-        public const string DiscoveryPortTag = "disc_port";
-        public const string AccountsFilename = "accounts.csv";
-
-        public override string AppName => "geth";
-        public override string Image => "codexstorage/dist-tests-geth:sha-b788a2d";
-
-        protected override void Initialize(StartupConfig startupConfig)
-        {
-            var config = startupConfig.Get<GethStartupConfig>();
-
-            var args = CreateArgs(config);
-
-            AddEnvVar("GETH_ARGS", args);
-        }
-
-        private string CreateArgs(GethStartupConfig config)
-        {
-            var discovery = AddInternalPort(tag: DiscoveryPortTag);
-
-            if (config.IsBootstrapNode)
-            {
-                return CreateBootstapArgs(discovery);
-            }
-
-            return CreateCompanionArgs(discovery, config);
-        }
-
-        private string CreateBootstapArgs(Port discovery)
-        {
-            AddEnvVar("ENABLE_MINER", "1");
-            UnlockAccounts(0, 1);
-            var exposedPort = AddExposedPort(tag: HttpPortTag);
-            return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}";
-        }
-
-        private string CreateCompanionArgs(Port discovery, GethStartupConfig config)
-        {
-            UnlockAccounts(
-                config.CompanionAccountStartIndex + 1,
-                config.NumberOfCompanionAccounts);
-
-            var port = AddInternalPort();
-            var authRpc = AddInternalPort();
-            var httpPort = AddExposedPort(tag: HttpPortTag);
-
-            var bootPubKey = config.BootstrapNode.PubKey;
-            var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.PodInfo.Ip;
-            var bootPort = config.BootstrapNode.DiscoveryPort.Number;
-            var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}";
-
-            return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}";
-        }
-
-        private void UnlockAccounts(int startIndex, int numberOfAccounts)
-        {
-            if (startIndex < 0) throw new ArgumentException();
-            if (numberOfAccounts < 1) throw new ArgumentException();
-            if (startIndex + numberOfAccounts > 1000) throw new ArgumentException("Out of accounts!");
-
-            AddEnvVar("UNLOCK_START_INDEX", startIndex.ToString());
-            AddEnvVar("UNLOCK_NUMBER", numberOfAccounts.ToString());
-        }
-    }
-}
@@ -1,19 +0,0 @@
-using Newtonsoft.Json;
-
-namespace DistTestCore.Marketplace
-{
-    public class GethStartResult
-    {
-        public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
-        {
-            MarketplaceAccessFactory = marketplaceAccessFactory;
-            MarketplaceNetwork = marketplaceNetwork;
-            CompanionNode = companionNode;
-        }
-
-        [JsonIgnore]
-        public IMarketplaceAccessFactory MarketplaceAccessFactory { get; }
-        public MarketplaceNetwork MarketplaceNetwork { get; }
-        public GethCompanionNodeInfo CompanionNode { get; }
-    }
-}
@@ -1,18 +0,0 @@
-namespace DistTestCore.Marketplace
-{
-    public class GethStartupConfig
-    {
-        public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int companionAccountStartIndex, int numberOfCompanionAccounts)
-        {
-            IsBootstrapNode = isBootstrapNode;
-            BootstrapNode = bootstrapNode;
-            CompanionAccountStartIndex = companionAccountStartIndex;
-            NumberOfCompanionAccounts = numberOfCompanionAccounts;
-        }
-
-        public bool IsBootstrapNode { get; }
-        public GethBootstrapNodeInfo BootstrapNode { get; }
-        public int CompanionAccountStartIndex { get; }
-        public int NumberOfCompanionAccounts { get; }
-    }
-}
@@ -1,243 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Helpers;
-using Logging;
-using Newtonsoft.Json;
-using NUnit.Framework;
-using NUnit.Framework.Constraints;
-using System.Numerics;
-using Utils;
-
-namespace DistTestCore.Marketplace
-{
-    public interface IMarketplaceAccess
-    {
-        string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration);
-        StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
-        void AssertThatBalance(IResolveConstraint constraint, string message = "");
-        TestToken GetBalance();
-    }
-
-    public class MarketplaceAccess : IMarketplaceAccess
-    {
-        private readonly TestLifecycle lifecycle;
-        private readonly MarketplaceNetwork marketplaceNetwork;
-        private readonly GethAccount account;
-        private readonly CodexAccess codexAccess;
-
-        public MarketplaceAccess(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
-        {
-            this.lifecycle = lifecycle;
-            this.marketplaceNetwork = marketplaceNetwork;
-            this.account = account;
-            this.codexAccess = codexAccess;
-        }
-
-        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
-        {
-            var request = new CodexSalesRequestStorageRequest
-            {
-                duration = ToDecInt(duration.TotalSeconds),
-                proofProbability = ToDecInt(proofProbability),
-                reward = ToDecInt(pricePerSlotPerSecond),
-                collateral = ToDecInt(requiredCollateral),
-                expiry = null,
-                nodes = minRequiredNumberOfNodes,
-                tolerance = null,
-            };
-
-            Log($"Requesting storage for: {contentId.Id}... (" +
-                $"pricePerSlotPerSecond: {pricePerSlotPerSecond}, " +
-                $"requiredCollateral: {requiredCollateral}, " +
-                $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
-                $"proofProbability: {proofProbability}, " +
-                $"duration: {Time.FormatDuration(duration)})");
-
-            var response = codexAccess.RequestStorage(request, contentId.Id);
-
-            if (response == "Purchasing not available")
-            {
-                throw new InvalidOperationException(response);
-            }
-
-            Log($"Storage requested successfully. PurchaseId: '{response}'.");
-
-            return new StoragePurchaseContract(lifecycle.Log, codexAccess, response, duration);
-        }
-
-        public string MakeStorageAvailable(ByteSize totalSpace, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration)
-        {
-            var request = new CodexSalesAvailabilityRequest
-            {
-                size = ToDecInt(totalSpace.SizeInBytes),
-                duration = ToDecInt(maxDuration.TotalSeconds),
-                maxCollateral = ToDecInt(maxCollateral),
-                minPrice = ToDecInt(minPriceForTotalSpace)
-            };
-
-            Log($"Making storage available... (" +
-                $"size: {totalSpace}, " +
-                $"minPriceForTotalSpace: {minPriceForTotalSpace}, " +
-                $"maxCollateral: {maxCollateral}, " +
-                $"maxDuration: {Time.FormatDuration(maxDuration)})");
-
-            var response = codexAccess.SalesAvailability(request);
-
-            Log($"Storage successfully made available. Id: {response.id}");
-
-            return response.id;
-        }
-
-        private string ToDecInt(double d)
-        {
-            var i = new BigInteger(d);
-            return i.ToString("D");
-        }
-
-        public string ToDecInt(TestToken t)
-        {
-            var i = new BigInteger(t.Amount);
-            return i.ToString("D");
-        }
-
-        public void AssertThatBalance(IResolveConstraint constraint, string message = "")
-        {
-            AssertHelpers.RetryAssert(constraint, GetBalance, message);
-        }
-
-        public TestToken GetBalance()
-        {
-            var interaction = marketplaceNetwork.StartInteraction(lifecycle);
-            var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account.Account);
-            var balance = new TestToken(amount);
-
-            Log($"Balance of {account.Account} is {balance}.");
-
-            return balance;
-        }
-
-        private void Log(string msg)
-        {
-            lifecycle.Log.Log($"{codexAccess.Container.Name} {msg}");
-        }
-    }
-
-    public class MarketplaceUnavailable : IMarketplaceAccess
-    {
-        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
-        {
-            Unavailable();
-            return null!;
-        }
-
-        public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration)
-        {
-            Unavailable();
-            return string.Empty;
-        }
-
-        public void AssertThatBalance(IResolveConstraint constraint, string message = "")
-        {
-            Unavailable();
-        }
-
-        public TestToken GetBalance()
-        {
-            Unavailable();
-            return new TestToken(0);
-        }
-
-        private void Unavailable()
-        {
-            Assert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it.");
-            throw new InvalidOperationException();
-        }
-    }
-
-    public class StoragePurchaseContract
-    {
-        private readonly BaseLog log;
-        private readonly CodexAccess codexAccess;
-        private DateTime? contractStartUtc;
-
-        public StoragePurchaseContract(BaseLog log, CodexAccess codexAccess, string purchaseId, TimeSpan contractDuration)
-        {
-            this.log = log;
-            this.codexAccess = codexAccess;
-            PurchaseId = purchaseId;
-            ContractDuration = contractDuration;
-        }
-
-        public string PurchaseId { get; }
-        public TimeSpan ContractDuration { get; }
-
-        public void WaitForStorageContractStarted()
-        {
-            WaitForStorageContractStarted(TimeSpan.FromSeconds(30));
-        }
-
-        public void WaitForStorageContractFinished()
-        {
-            if (!contractStartUtc.HasValue)
-            {
-                WaitForStorageContractStarted();
-            }
-            var gracePeriod = TimeSpan.FromSeconds(10);
-            var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
-            var timeout = (ContractDuration - currentContractTime) + gracePeriod;
-            WaitForStorageContractState(timeout, "finished");
-        }
-
-        /// <summary>
-        /// Wait for contract to start. Max timeout depends on contract filesize. Allows more time for larger files.
-        /// </summary>
-        public void WaitForStorageContractStarted(ByteSize contractFileSize)
-        {
-            var filesizeInMb = contractFileSize.SizeInBytes / (1024 * 1024);
-            var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0);
-
-            WaitForStorageContractStarted(maxWaitTime);
-        }
-
-        public void WaitForStorageContractStarted(TimeSpan timeout)
-        {
-            WaitForStorageContractState(timeout, "started");
-            contractStartUtc = DateTime.UtcNow;
-        }
-
-        private void WaitForStorageContractState(TimeSpan timeout, string desiredState)
-        {
-            var lastState = "";
-            var waitStart = DateTime.UtcNow;
-            
-            log.Log($"Waiting for {Time.FormatDuration(timeout)} for contract '{PurchaseId}' to reach state '{desiredState}'.");
-            while (lastState != desiredState)
-            {
-                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
-                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
-                if (purchaseStatus != null && purchaseStatus.state != lastState)
-                {
-                    lastState = purchaseStatus.state;
-                    log.Debug("Purchase status: " + statusJson);
-                }
-
-                Thread.Sleep(1000);
-
-                if (lastState == "errored")
-                {
-                    Assert.Fail("Contract errored: " + statusJson);
-                }
-
-                if (DateTime.UtcNow - waitStart > timeout)
-                {
-                    Assert.Fail($"Contract did not reach '{desiredState}' within timeout. {statusJson}");
-                }
-            }
-            log.Log($"Contract '{desiredState}'.");
-        }
-
-        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
-        {
-            return codexAccess.GetPurchaseStatus(purchaseId);
-        }
-    }
-}
@@ -1,41 +0,0 @@
-using DistTestCore.Codex;
-
-namespace DistTestCore.Marketplace
-{
-    public interface IMarketplaceAccessFactory
-    {
-        IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access);
-    }
-
-    public class MarketplaceUnavailableAccessFactory : IMarketplaceAccessFactory
-    {
-        public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
-        {
-            return new MarketplaceUnavailable();
-        }
-    }
-
-    public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory
-    {
-        private readonly TestLifecycle lifecycle;
-        private readonly MarketplaceNetwork marketplaceNetwork;
-
-        public GethMarketplaceAccessFactory(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork)
-        {
-            this.lifecycle = lifecycle;
-            this.marketplaceNetwork = marketplaceNetwork;
-        }
-
-        public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
-        {
-            var companionNode = GetGethCompanionNode(access);
-            return new MarketplaceAccess(lifecycle, marketplaceNetwork, companionNode, access);
-        }
-
-        private GethAccount GetGethCompanionNode(CodexAccess access)
-        {
-            var account = access.Container.Recipe.Additionals.Single(a => a is GethAccount);
-            return (GethAccount)account;
-        }
-    }
-}
@@ -1,17 +0,0 @@
-namespace DistTestCore.Marketplace
-{
-    public class MarketplaceInitialConfig
-    {
-        public MarketplaceInitialConfig(Ether initialEth, TestToken initialTestTokens, bool isValidator)
-        {
-            InitialEth = initialEth;
-            InitialTestTokens = initialTestTokens;
-            IsValidator = isValidator;
-        }
-
-        public Ether InitialEth { get; }
-        public TestToken InitialTestTokens { get; }
-        public bool IsValidator { get; }
-        public int? AccountIndexOverride { get; set; }
-    }
-}
@@ -1,21 +0,0 @@
-using NethereumWorkflow;
-
-namespace DistTestCore.Marketplace
-{
-    public class MarketplaceNetwork
-    {
-        public MarketplaceNetwork(GethBootstrapNodeInfo bootstrap, MarketplaceInfo marketplace)
-        {
-            Bootstrap = bootstrap;
-            Marketplace = marketplace;
-        }
-
-        public GethBootstrapNodeInfo Bootstrap { get; }
-        public MarketplaceInfo Marketplace { get; }
-
-        public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
-        {
-            return Bootstrap.StartInteraction(lifecycle);
-        }
-    }
-}
@@ -1,81 +0,0 @@
-using DistTestCore.Helpers;
-using KubernetesWorkflow;
-using Logging;
-using NUnit.Framework;
-using NUnit.Framework.Constraints;
-using Utils;
-
-namespace DistTestCore.Metrics
-{
-    public interface IMetricsAccess
-    {
-        void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
-    }
-
-    public class MetricsAccess : IMetricsAccess
-    {
-        private readonly BaseLog log;
-        private readonly ITimeSet timeSet;
-        private readonly MetricsQuery query;
-        private readonly RunningContainer node;
-
-        public MetricsAccess(BaseLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
-        {
-            this.log = log;
-            this.timeSet = timeSet;
-            this.query = query;
-            this.node = node;
-        }
-
-        public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
-        {
-            AssertHelpers.RetryAssert(constraint, () =>
-            {
-                var metricSet = GetMetricWithTimeout(metricName);
-                var metricValue = metricSet.Values[0].Value;
-
-                log.Log($"{node.Name} metric '{metricName}' = {metricValue}");
-                return metricValue;
-            }, message);
-        }
-
-        public Metrics? GetAllMetrics()
-        {
-            return query.GetAllMetricsForNode(node);
-        }
-
-        private MetricsSet GetMetricWithTimeout(string metricName)
-        {
-            var start = DateTime.UtcNow;
-
-            while (true)
-            {
-                var mostRecent = GetMostRecent(metricName);
-                if (mostRecent != null) return mostRecent;
-                if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout())
-                {
-                    Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
-                    throw new TimeoutException();
-                }
-
-                Time.Sleep(TimeSpan.FromSeconds(2));
-            }
-        }
-
-        private MetricsSet? GetMostRecent(string metricName)
-        {
-            var result = query.GetMostRecent(metricName, node);
-            if (result == null) return null;
-            return result.Sets.LastOrDefault();
-        }
-    }
-
-    public class MetricsUnavailable : IMetricsAccess
-    {
-        public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
-        {
-            Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
-            throw new InvalidOperationException();
-        }
-    }
-}
@@ -1,35 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore.Metrics
-{
-    public interface IMetricsAccessFactory
-    {
-        IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer);
-    }
-
-    public class MetricsUnavailableAccessFactory : IMetricsAccessFactory
-    {
-        public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
-        {
-            return new MetricsUnavailable();
-        }
-    }
-
-    public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory
-    {
-        private readonly TestLifecycle lifecycle;
-        private readonly RunningContainers prometheusContainer;
-
-        public CodexNodeMetricsAccessFactory(TestLifecycle lifecycle, RunningContainers prometheusContainer)
-        {
-            this.lifecycle = lifecycle;
-            this.prometheusContainer = prometheusContainer;
-        }
-
-        public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
-        {
-            var query = new MetricsQuery(lifecycle, prometheusContainer);
-            return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
-        }
-    }
-}
@@ -1,173 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Logs;
-using DistTestCore.Marketplace;
-using DistTestCore.Metrics;
-using Logging;
-using NUnit.Framework;
-
-namespace DistTestCore
-{
-    public interface IOnlineCodexNode
-    {
-        string GetName();
-        CodexDebugResponse GetDebugInfo();
-        CodexDebugPeerResponse GetDebugPeer(string peerId);
-        ContentId UploadFile(TestFile file);
-        TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
-        void ConnectToPeer(IOnlineCodexNode node);
-        IDownloadedLog DownloadLog();
-        IMetricsAccess Metrics { get; }
-        IMarketplaceAccess Marketplace { get; }
-        CodexDebugVersionResponse Version { get; }
-        ICodexSetup BringOffline();
-    }
-
-    public class OnlineCodexNode : IOnlineCodexNode
-    {
-        private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
-        private const string UploadFailedMessage = "Unable to store block";
-        private readonly TestLifecycle lifecycle;
-
-        public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess)
-        {
-            this.lifecycle = lifecycle;
-            CodexAccess = codexAccess;
-            Group = group;
-            Metrics = metricsAccess;
-            Marketplace = marketplaceAccess;
-            Version = new CodexDebugVersionResponse();
-        }
-
-        public CodexAccess CodexAccess { get; }
-        public CodexNodeGroup Group { get; }
-        public IMetricsAccess Metrics { get; }
-        public IMarketplaceAccess Marketplace { get; }
-        public CodexDebugVersionResponse Version { get; private set; }
-
-        public string GetName()
-        {
-            return CodexAccess.Container.Name;
-        }
-
-        public CodexDebugResponse GetDebugInfo()
-        {
-            var debugInfo = CodexAccess.GetDebugInfo();
-            var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
-            Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
-            return debugInfo;
-        }
-
-        public CodexDebugPeerResponse GetDebugPeer(string peerId)
-        {
-            return CodexAccess.GetDebugPeer(peerId);
-        }
-
-        public ContentId UploadFile(TestFile file)
-        {
-            using var fileStream = File.OpenRead(file.Filename);
-
-            var logMessage = $"Uploading file {file.Describe()}...";
-            var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
-            {
-                return CodexAccess.UploadFile(fileStream);
-            });
-
-            if (string.IsNullOrEmpty(response)) Assert.Fail("Received empty response.");
-            if (response.StartsWith(UploadFailedMessage)) Assert.Fail("Node failed to store block.");
-
-            Log($"Uploaded file. Received contentId: '{response}'.");
-            return new ContentId(response);
-        }
-
-        public TestFile? DownloadContent(ContentId contentId, string fileLabel = "")
-        {
-            var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
-            var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel);
-            Stopwatch.Measure(lifecycle.Log, logMessage, () => DownloadToFile(contentId.Id, file));
-            Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
-            return file;
-        }
-
-        public void ConnectToPeer(IOnlineCodexNode node)
-        {
-            var peer = (OnlineCodexNode)node;
-
-            Log($"Connecting to peer {peer.GetName()}...");
-            var peerInfo = node.GetDebugInfo();
-            var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
-
-            Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes.");
-            Log($"Successfully connected to peer {peer.GetName()}.");
-        }
-
-        public IDownloadedLog DownloadLog()
-        {
-            return lifecycle.DownloadLog(CodexAccess.Container);
-        }
-
-        public ICodexSetup BringOffline()
-        {
-            if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
-                "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
-                "available for codex-nodes in groups of 1.");
-
-            return Group.BringOffline();
-        }
-
-        public void EnsureOnlineGetVersionResponse()
-        {
-            var debugInfo = CodexAccess.GetDebugInfo();
-            var nodePeerId = debugInfo.id;
-            var nodeName = CodexAccess.Container.Name;
-
-            if (!debugInfo.codex.IsValid())
-            {
-                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}");
-            }
-
-            lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
-            lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
-            Version = debugInfo.codex;
-        }
-
-        private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
-        {
-            var multiAddress = peerInfo.addrs.First();
-            // Todo: Is there a case where First address in list is not the way?
-
-            // The peer we want to connect is in a different pod.
-            // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip);
-        }
-
-        private void DownloadToFile(string contentId, TestFile file)
-        {
-            using var fileStream = File.OpenWrite(file.Filename);
-            try
-            {
-                using var downloadStream = CodexAccess.DownloadFile(contentId);
-                downloadStream.CopyTo(fileStream);
-            }
-            catch
-            {
-                Log($"Failed to download file '{contentId}'.");
-                throw;
-            }
-        }
-
-        private void Log(string msg)
-        {
-            lifecycle.Log.Log($"{GetName()}: {msg}");
-        }
-    }
-
-    public class ContentId
-    {
-        public ContentId(string id)
-        {
-            Id = id;
-        }
-
-        public string Id { get; }
-    }
-}
@@ -1,54 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Metrics;
-using KubernetesWorkflow;
-using System.Text;
-
-namespace DistTestCore
-{
-    public class PrometheusStarter : BaseStarter
-    {
-        public PrometheusStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
-        {
-        }
-
-        public RunningContainers CollectMetricsFor(RunningContainers[] containers)
-        {
-            LogStart($"Starting metrics server for {containers.Describe()}");
-            var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers())));
-
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
-            var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
-            if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
-
-            LogEnd("Metrics server started.");
-
-            return runningContainers;
-        }
-
-        private string GeneratePrometheusConfig(RunningContainer[] nodes)
-        {
-            var config = "";
-            config += "global:\n";
-            config += "  scrape_interval: 30s\n";
-            config += "  scrape_timeout: 10s\n";
-            config += "\n";
-            config += "scrape_configs:\n";
-            config += "  - job_name: services\n";
-            config += "    metrics_path: /metrics\n";
-            config += "    static_configs:\n";
-            config += "      - targets:\n";
-
-            foreach (var node in nodes)
-            {
-                var ip = node.Pod.PodInfo.Ip;
-                var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
-                config += $"          - '{ip}:{port}'\n";
-            }
-
-            var bytes = Encoding.ASCII.GetBytes(config);
-            return Convert.ToBase64String(bytes);
-        }
-    }
-}
@@ -1,91 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Logs;
-using DistTestCore.Marketplace;
-using DistTestCore.Metrics;
-using KubernetesWorkflow;
-using Logging;
-using Utils;
-
-namespace DistTestCore
-{
-    public class TestLifecycle
-    {
-        private readonly DateTime testStart;
-
-        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testsType, string testNamespace)
-        {
-            Log = log;
-            Configuration = configuration;
-            TimeSet = timeSet;
-
-            var podLabels = new PodLabels(testsType, GetApplicationIds());
-            WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), podLabels, testNamespace);
-
-            FileManager = new FileManager(Log, configuration);
-            CodexStarter = new CodexStarter(this);
-            PrometheusStarter = new PrometheusStarter(this);
-            GethStarter = new GethStarter(this);
-            testStart = DateTime.UtcNow;
-            CodexVersion = null;
-
-            Log.WriteLogTag();
-        }
-
-        public BaseLog Log { get; }
-        public Configuration Configuration { get; }
-        public ITimeSet TimeSet { get; }
-        public WorkflowCreator WorkflowCreator { get; }
-        public FileManager FileManager { get; }
-        public CodexStarter CodexStarter { get; }
-        public PrometheusStarter PrometheusStarter { get; }
-        public GethStarter GethStarter { get; }
-        public CodexDebugVersionResponse? CodexVersion { get; private set; }
-
-        public void DeleteAllResources()
-        {
-            CodexStarter.DeleteAllResources();
-            FileManager.DeleteAllTestFiles();
-        }
-
-        public IDownloadedLog DownloadLog(RunningContainer container)
-        {
-            var subFile = Log.CreateSubfile();
-            var description = container.Name;
-            var handler = new LogDownloadHandler(container, description, subFile);
-
-            Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-            CodexStarter.DownloadLog(container, handler);
-
-            return new DownloadedLog(subFile, description);
-        }
-
-        public string GetTestDuration()
-        {
-            var testDuration = DateTime.UtcNow - testStart;
-            return Time.FormatDuration(testDuration);
-        }
-
-        public void SetCodexVersion(CodexDebugVersionResponse version)
-        {
-            if (CodexVersion == null) CodexVersion = version;
-        }
-
-        public ApplicationIds GetApplicationIds()
-        {
-            return new ApplicationIds(
-                codexId: GetCodexId(),
-                gethId: new GethContainerRecipe().Image,
-                prometheusId: new PrometheusContainerRecipe().Image,
-                codexContractsId: new CodexContractsContainerRecipe().Image
-            );
-        }
-
-        private string GetCodexId()
-        {
-            var v = CodexVersion;
-            if (v == null) return new CodexContainerRecipe().Image;
-            if (v.version != "untagged build") return v.version;
-            return v.revision;
-        }
-    }
-}
@@ -1,85 +0,0 @@
-using NUnit.Framework;
-
-namespace DistTestCore
-{
-    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
-    public class UseLongTimeoutsAttribute : PropertyAttribute
-    {
-    }
-
-    public interface ITimeSet
-    {
-        TimeSpan HttpCallTimeout();
-        TimeSpan HttpCallRetryTime();
-        TimeSpan HttpCallRetryDelay();
-        TimeSpan WaitForK8sServiceDelay();
-        TimeSpan K8sOperationTimeout();
-        TimeSpan WaitForMetricTimeout();
-    }
-
-    public class DefaultTimeSet : ITimeSet
-    {
-        public TimeSpan HttpCallTimeout()
-        {
-            return TimeSpan.FromSeconds(10);
-        }
-
-        public TimeSpan HttpCallRetryTime()
-        {
-            return TimeSpan.FromMinutes(1);
-        }
-
-        public TimeSpan HttpCallRetryDelay()
-        {
-            return TimeSpan.FromSeconds(1);
-        }
-
-        public TimeSpan WaitForK8sServiceDelay()
-        {
-            return TimeSpan.FromSeconds(1);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromMinutes(1);
-        }
-
-        public TimeSpan WaitForMetricTimeout()
-        {
-            return TimeSpan.FromSeconds(30);
-        }
-    }
-
-    public class LongTimeSet : ITimeSet
-    {
-        public TimeSpan HttpCallTimeout()
-        {
-            return TimeSpan.FromHours(2);
-        }
-
-        public TimeSpan HttpCallRetryTime()
-        {
-            return TimeSpan.FromHours(5);
-        }
-
-        public TimeSpan HttpCallRetryDelay()
-        {
-            return TimeSpan.FromSeconds(2);
-        }
-
-        public TimeSpan WaitForK8sServiceDelay()
-        {
-            return TimeSpan.FromSeconds(10);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromMinutes(15);
-        }
-
-        public TimeSpan WaitForMetricTimeout()
-        {
-            return TimeSpan.FromMinutes(5);
-        }
-    }
-}
@@ -1,97 +0,0 @@
-namespace DistTestCore
-{
-    public class Ether : IComparable<Ether>
-    {
-        public Ether(decimal wei)
-        {
-            Wei = wei;
-        }
-
-        public decimal Wei { get; }
-
-        public int CompareTo(Ether? other)
-        {
-            return Wei.CompareTo(other!.Wei);
-        }
-
-        public override bool Equals(object? obj)
-        {
-            return obj is Ether ether && Wei == ether.Wei;
-        }
-
-        public override int GetHashCode()
-        {
-            return HashCode.Combine(Wei);
-        }
-
-        public override string ToString()
-        {
-            return $"{Wei} Wei";
-        }
-    }
-
-    public class TestToken : IComparable<TestToken>
-    {
-        public TestToken(decimal amount)
-        {
-            Amount = amount;
-        }
-
-        public decimal Amount { get; }
-
-        public int CompareTo(TestToken? other)
-        {
-            return Amount.CompareTo(other!.Amount);
-        }
-
-        public override bool Equals(object? obj)
-        {
-            return obj is TestToken token && Amount == token.Amount;
-        }
-
-        public override int GetHashCode()
-        {
-            return HashCode.Combine(Amount);
-        }
-
-        public override string ToString()
-        {
-            return $"{Amount} TestTokens";
-        }
-    }
-
-    public static class TokensIntExtensions
-    {
-        private const decimal weiPerEth = 1000000000000000000;
-
-        public static TestToken TestTokens(this int i)
-        {
-            return TestTokens(Convert.ToDecimal(i));
-        }
-
-        public static TestToken TestTokens(this decimal i)
-        {
-            return new TestToken(i);
-        }
-
-        public static Ether Eth(this int i)
-        {
-            return Eth(Convert.ToDecimal(i));
-        }
-
-        public static Ether Wei(this int i)
-        {
-            return Wei(Convert.ToDecimal(i));
-        }
-
-        public static Ether Eth(this decimal i)
-        {
-            return new Ether(i * weiPerEth);
-        }
-
-        public static Ether Wei(this decimal i)
-        {
-            return new Ether(i);
-        }
-    }
-}
@@ -0,0 +1,130 @@
+using System.Reflection;
+
+namespace ArgsUniform
+{
+    public class ArgsUniform<T>
+    {
+        private readonly Assigner<T> assigner;
+        private readonly Action printAppInfo;
+        private readonly string[] args;
+        private const int cliStart = 8;
+        private const int shortStart = 38;
+        private const int envStart = 48;
+        private const int descStart = 80;
+
+        public ArgsUniform(Action printAppInfo, params string[] args)
+            : this(printAppInfo, new IEnv.Env(), args)
+        {
+        }
+
+        public ArgsUniform(Action printAppInfo, object defaultsProvider, params string[] args)
+            : this(printAppInfo, defaultsProvider, new IEnv.Env(), args)
+        {
+        }
+
+        public ArgsUniform(Action printAppInfo, IEnv.IEnv env, params string[] args)
+            : this(printAppInfo, null!, env, args)
+        {
+        }
+
+        public ArgsUniform(Action printAppInfo, object defaultsProvider, IEnv.IEnv env, params string[] args)
+        {
+            this.printAppInfo = printAppInfo;
+            this.args = args;
+
+            assigner = new Assigner<T>(env, args, defaultsProvider);
+        }
+
+        public T Parse(bool printResult = false)
+        {
+            if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
+            {
+                printAppInfo();
+                PrintHelp();
+                Environment.Exit(0);
+            }
+
+            var result = Activator.CreateInstance<T>();
+            var uniformProperties = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
+            var missingRequired = new List<PropertyInfo>();
+            foreach (var uniformProperty in uniformProperties)
+            {
+                var attr = uniformProperty.GetCustomAttribute<UniformAttribute>();
+                if (attr != null)
+                {
+                    if (!assigner.UniformAssign(result, attr, uniformProperty) && attr.Required)
+                    {
+                        missingRequired.Add(uniformProperty);
+                    }
+                }
+            }
+
+            if (missingRequired.Any())
+            {
+                PrintResults(printResult,result, uniformProperties);
+                Print("");
+                foreach (var missing in missingRequired)
+                {
+                    var attr = missing.GetCustomAttribute<UniformAttribute>()!;
+                    var exampleArg = $"--{attr.Arg}=...";
+                    var exampleEnvVar = $"{attr.EnvVar}=...";
+                    Print($" ! Missing required input. Use argument: '{exampleArg}' or environment variable: '{exampleEnvVar}'.");
+                }
+
+                PrintHelp();
+                Environment.Exit(1);
+            }
+
+            PrintResults(printResult, result, uniformProperties);
+
+            return result;
+        }
+
+        public void PrintHelp()
+        {
+            Print("");
+            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description", "(default)");
+            var props = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
+            foreach (var prop in props)
+            {
+                var a = prop.GetCustomAttribute<UniformAttribute>();
+                if (a != null)
+                {
+                    var optional = !a.Required ? " (optional)" : "";
+                    var def = assigner.DescribeDefaultFor(prop);
+                    PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional, $"({def})");
+                }
+            }
+            Print("");
+        }
+
+        private void PrintResults(bool printResult, T result, PropertyInfo[] uniformProperties)
+        {
+            if (!printResult) return;
+            Print("");
+            foreach (var p in uniformProperties)
+            {
+                Print($"\t{p.Name} = {p.GetValue(result)}");
+            }
+            Print("");
+        }
+
+        private void Print(string msg)
+        {
+            Console.WriteLine(msg);
+        }
+
+        private void PrintAligned(string cli, string s, string env, string desc, string def)
+        {
+            Console.CursorLeft = cliStart;
+            Console.Write(cli);
+            Console.CursorLeft = shortStart;
+            Console.Write(s);
+            Console.CursorLeft = envStart;
+            Console.Write(env);
+            Console.CursorLeft = descStart;
+            Console.Write(desc + " ");
+            Console.Write(def + Environment.NewLine);
+        }
+    }
+}
@@ -0,0 +1,186 @@
+using System.Globalization;
+using System.Numerics;
+using System.Reflection;
+
+namespace ArgsUniform
+{
+    public class Assigner<T>
+    {
+        private readonly IEnv.IEnv env;
+        private readonly string[] args;
+        private readonly object? defaultsProvider;
+
+        public Assigner(IEnv.IEnv env, string[] args, object? defaultsProvider)
+        {
+            this.env = env;
+            this.args = args;
+            this.defaultsProvider = defaultsProvider;
+        }
+
+        public bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
+            return false;
+        }
+
+        public string DescribeDefaultFor(PropertyInfo property)
+        {
+            var obj = Activator.CreateInstance<T>();
+            var defaultValue = GetDefaultValue(obj, property);
+            if (defaultValue == null) return "";
+            if (defaultValue is string str)
+            {
+                return "\"" + str + "\"";
+            }
+            return defaultValue.ToString() ?? string.Empty;
+        }
+
+        private object? GetDefaultValue(T result, PropertyInfo uniformProperty)
+        {
+            // Get value from object's static initializer if it's there.
+            var currentValue = uniformProperty.GetValue(result);
+            if (currentValue != null) return currentValue;
+
+            // Get value from defaults-provider object if it's there.
+            if (defaultsProvider == null) return null;
+            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
+            if (defaultProperty == null) return null;
+            return defaultProperty.GetValue(defaultsProvider);
+        }
+
+        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
+        {
+            var defaultValue = GetDefaultValue(result, uniformProperty);
+            var isEmptryString = (defaultValue as string) == string.Empty;
+            if (defaultValue != null && defaultValue != GetDefaultValueForType(uniformProperty.PropertyType) && !isEmptryString)
+            {
+                return Assign(result, uniformProperty, defaultValue);
+            }
+            return false;
+        }
+
+        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
+            if (!string.IsNullOrEmpty(e))
+            {
+                return Assign(result, uniformProperty, e);
+            }
+            return false;
+        }
+
+        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var fromArg = GetFromArgs(attr.Arg);
+            if (fromArg != null)
+            {
+                return Assign(result, uniformProperty, fromArg);
+            }
+            var fromShort = GetFromArgs(attr.ArgShort);
+            if (fromShort != null)
+            {
+                return Assign(result, uniformProperty, fromShort);
+            }
+            return false;
+        }
+
+        private bool Assign(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (uniformProperty.PropertyType == value.GetType())
+            {
+                uniformProperty.SetValue(result, value);
+                return true;
+            }
+            else
+            {
+                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
+                {
+                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
+                    return true;
+                }
+                else
+                {
+                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(ulong)) return AssignUlong(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(BigInteger)) return AssignBigInt(result, uniformProperty, value);
+
+                    throw new NotSupportedException(
+                        $"Unsupported property type '${uniformProperty.PropertyType}' " +
+                        $"for property '${uniformProperty.Name}'.");
+                }
+            }
+        }
+
+        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
+            {
+                uniformProperty.SetValue(result, e);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (int.TryParse(value.ToString(), CultureInfo.InvariantCulture, out int i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignUlong(T? result, PropertyInfo uniformProperty, object value)
+        {
+            if (ulong.TryParse(value.ToString(), CultureInfo.InvariantCulture, out ulong i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignBigInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (BigInteger.TryParse(value.ToString(), CultureInfo.InvariantCulture, out BigInteger i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
+            {
+                uniformProperty.SetValue(result, true);
+            }
+            return true;
+        }
+
+        private string? GetFromArgs(string key)
+        {
+            var argKey = $"--{key}=";
+            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
+            if (arg != null)
+            {
+                return arg.Substring(argKey.Length);
+            }
+            return null;
+        }
+
+        private static object GetDefaultValueForType(Type t)
+        {
+            if (t.IsValueType) return Activator.CreateInstance(t)!;
+            return null!;
+        }
+    }
+}
@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\FileUtils\FileUtils.csproj" />
+    <ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,52 @@
+using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
+
+namespace Core
+{
+    public sealed class CoreInterface
+    {
+        private readonly EntryPoint entryPoint;
+
+        internal CoreInterface(EntryPoint entryPoint)
+        {
+            this.entryPoint = entryPoint;
+        }
+
+        public T GetPlugin<T>() where T : IProjectPlugin
+        {
+            return entryPoint.GetPlugin<T>();
+        }
+
+        public IKnownLocations GetKnownLocations()
+        {
+            return entryPoint.Tools.CreateWorkflow().GetAvailableLocations();
+        }
+
+        public IDownloadedLog DownloadLog(IHasContainer containerSource, int? tailLines = null)
+        {
+            return DownloadLog(containerSource.Container, tailLines);
+        }
+
+        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
+        {
+            var workflow = entryPoint.Tools.CreateWorkflow();
+            return workflow.DownloadContainerLog(container, tailLines);
+        }
+
+        public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)
+        {
+            return ExecuteContainerCommand(containerSource.Container, command, args);
+        }
+
+        public string ExecuteContainerCommand(RunningContainer container, string command, params string[] args)
+        {
+            var workflow = entryPoint.Tools.CreateWorkflow();
+            return workflow.ExecuteCommand(container, command, args);
+        }
+    }
+
+    public interface IHasContainer
+    {
+        RunningContainer Container { get; }
+    }
+}
@@ -0,0 +1,193 @@
+using Logging;
+using Newtonsoft.Json;
+using Serialization = Newtonsoft.Json.Serialization;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using Utils;
+
+namespace Core
+{
+    public interface IEndpoint
+    {
+        string HttpGetString(string route);
+        T HttpGetJson<T>(string route);
+        TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
+        string HttpPostJson<TRequest>(string route, TRequest body);
+        TResponse HttpPostString<TResponse>(string route, string body);
+        string HttpPostStream(string route, Stream stream);
+        Stream HttpGetStream(string route);
+        T Deserialize<T>(string json);
+    }
+
+    internal class Endpoint : IEndpoint
+    {
+        private readonly ILog log;
+        private readonly IHttp http;
+        private readonly Address address;
+        private readonly string baseUrl;
+        private readonly string? logAlias;
+
+        public Endpoint(ILog log, IHttp http, Address address, string baseUrl, string? logAlias)
+        {
+            this.log = log;
+            this.http = http;
+            this.address = address;
+            this.baseUrl = baseUrl;
+            this.logAlias = logAlias;
+        }
+
+        public string HttpGetString(string route)
+        {
+            return http.OnClient(client =>
+            {
+                return GetString(client, route);
+            }, $"HTTP-GET:{route}");
+        }
+
+        public T HttpGetJson<T>(string route)
+        {
+            return http.OnClient(client =>
+            {
+                var json = GetString(client, route);
+                return Deserialize<T>(json);
+            }, $"HTTP-GET:{route}");
+        }
+
+        public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJson(client, route, body);
+                var json = Time.Wait(response.Content.ReadAsStringAsync());
+                if (!response.IsSuccessStatusCode)
+                {
+                    throw new HttpRequestException(json);
+                }
+                Log(GetUrl() + route, json);
+                return Deserialize<TResponse>(json);
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        public string HttpPostJson<TRequest>(string route, TRequest body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJson(client, route, body);
+                return Time.Wait(response.Content.ReadAsStringAsync());
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        public TResponse HttpPostString<TResponse>(string route, string body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJsonString(client, route, body);
+                if (response == null) throw new Exception("Received no response.");
+                var result = Deserialize<TResponse>(response);
+                if (result == null) throw new Exception("Failed to deserialize response");
+                return result;
+            }, $"HTTO-POST-JSON: {route}");
+        }
+
+        public string HttpPostStream(string route, Stream stream)
+        {
+            return http.OnClient(client =>
+            {
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                var content = new StreamContent(stream);
+                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
+                var response = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(response.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STREAM: {route}");
+        }
+
+        public Stream HttpGetStream(string route)
+        {
+            return http.OnClient(client =>
+            {
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                return Time.Wait(client.GetStreamAsync(url));
+            }, $"HTTP-GET-STREAM: {route}");
+        }
+
+        public T Deserialize<T>(string json)
+        {
+            var errors = new List<string>();
+            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
+            {
+                Error = delegate (object? sender, Serialization.ErrorEventArgs args)
+                {
+                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
+                    {
+                        errors.Add($"""
+                                    Member: '{args.ErrorContext.Member?.ToString() ?? "<null>"}'
+                                    Path: {args.ErrorContext.Path}
+                                    Error: {args.ErrorContext.Error.Message}
+                                    """);
+                        args.ErrorContext.Handled = true;
+                    }
+                }
+            });
+            if (errors.Count > 0)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
+            }
+            else if (deserialized == null)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
+            }
+            return deserialized;
+        }
+
+        private string GetString(HttpClient client, string route)
+        {
+            var url = GetUrl() + route;
+            Log(url, "");
+            var result = Time.Wait(client.GetAsync(url));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
+        private HttpResponseMessage PostJson<TRequest>(HttpClient client, string route, TRequest body)
+        {
+            var url = GetUrl() + route;
+            using var content = JsonContent.Create(body);
+            Log(url, JsonConvert.SerializeObject(body));
+            return Time.Wait(client.PostAsync(url, content));
+        }
+
+        private string PostJsonString(HttpClient client, string route, string body)
+        {
+            var url = GetUrl() + route;
+            Log(url, body);
+            var content = new StringContent(body);
+            content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+            var result = Time.Wait(client.PostAsync(url, content));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
+        private string GetUrl()
+        {
+            return $"{address.Host}:{address.Port}{baseUrl}";
+        }
+
+        private void Log(string url, string message)
+        {
+            if (logAlias != null)
+            {
+                log.Debug($"({logAlias})({url}) = '{message}'", 3);
+            }
+            else
+            {
+                log.Debug($"({url}) = '{message}'", 3);
+            }
+        }
+    }
+}
@@ -0,0 +1,56 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace Core
+{
+    public class EntryPoint
+    {
+        private readonly IToolsFactory toolsFactory;
+        private readonly PluginManager manager = new PluginManager();
+
+        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        {
+            toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, timeSet);
+
+            Tools = toolsFactory.CreateTools();
+            manager.InstantiatePlugins(PluginFinder.GetPluginTypes(), toolsFactory);
+        }
+
+        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder)
+            : this(log, configuration, fileManagerRootFolder, new DefaultTimeSet())
+        {
+        }
+
+        public IPluginTools Tools { get; }
+
+        public void Announce()
+        {
+            manager.AnnouncePlugins();
+        }
+
+        public Dictionary<string, string> GetPluginMetadata()
+        {
+            return manager.GatherPluginMetadata().Get();
+        }
+
+        public CoreInterface CreateInterface()
+        {
+            return new CoreInterface(this);
+        }
+
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
+        {
+            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
+            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
+        }
+
+        internal T GetPlugin<T>() where T : IProjectPlugin
+        {
+            return manager.GetPlugin<T>();
+        }
+    }
+}
@@ -0,0 +1,97 @@
+using Logging;
+using Utils;
+
+namespace Core
+{
+    public interface IHttp
+    {
+        T OnClient<T>(Func<HttpClient, T> action);
+        T OnClient<T>(Func<HttpClient, T> action, string description);
+        T OnClient<T>(Func<HttpClient, T> action, Retry retry);
+        IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null);
+    }
+
+    internal class Http : IHttp
+    {
+        private static object lockLock = new object();
+        private static readonly Dictionary<string, object> httpLocks = new Dictionary<string, object>();
+        private readonly ILog log;
+        private readonly ITimeSet timeSet;
+        private readonly Action<HttpClient> onClientCreated;
+        private readonly string id;
+
+        internal Http(string id, ILog log, ITimeSet timeSet)
+            : this(id, log, timeSet, DoNothing)
+        {
+        }
+
+        internal Http(string id, ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
+        {
+            this.id = id;
+            this.log = log;
+            this.timeSet = timeSet;
+            this.onClientCreated = onClientCreated;
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action)
+        {
+            return OnClient(action, GetDescription());
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action, string description)
+        {
+            var retry = new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), f => { });
+            return OnClient(action, retry);
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action, Retry retry)
+        {
+            var client = GetClient();
+
+            return LockRetry(() =>
+            {
+                return action(client);
+            }, retry);
+        }
+
+        public IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null)
+        {
+            return new Endpoint(log, this, address, baseUrl, logAlias);
+        }
+
+        private string GetDescription()
+        {
+            return DebugStack.GetCallerName(skipFrames: 2);
+        }
+
+        private T LockRetry<T>(Func<T> operation, Retry retry)
+        {
+            var httpLock = GetLock();
+            lock (httpLock)
+            {
+                return retry.Run(operation);
+            }
+        }
+
+        private object GetLock()
+        {
+            lock (lockLock) // I had to.
+            {
+                if (!httpLocks.ContainsKey(id)) httpLocks.Add(id, new object());
+                return httpLocks[id];
+            }
+        }
+
+        private HttpClient GetClient()
+        {
+            var client = new HttpClient();
+            client.Timeout = timeSet.HttpCallTimeout();
+            onClientCreated(client);
+            return client;
+        }
+
+        private static void DoNothing(HttpClient client)
+        {
+        }
+    }
+}
@@ -0,0 +1,46 @@
+using System.Reflection;
+
+namespace Core
+{
+    internal static class PluginFinder
+    {
+        private static Type[]? pluginTypes = null;
+
+        internal static Type[] GetPluginTypes()
+        {
+            if (pluginTypes != null) return pluginTypes;
+
+            // Reflection can be costly. Do this only once.
+            FindAndLoadPluginAssemblies();
+
+            var assemblies = AppDomain.CurrentDomain.GetAssemblies();
+            pluginTypes = assemblies.SelectMany(a => a.GetTypes().Where(t =>
+                typeof(IProjectPlugin).IsAssignableFrom(t) &&
+                !t.IsAbstract)
+            ).ToArray();
+
+            return pluginTypes;
+        }
+
+        private static void FindAndLoadPluginAssemblies()
+        {
+            var files = Directory.GetFiles(".");
+            foreach (var file in files)
+            {
+                var f = file.ToLowerInvariant();
+                if (f.Contains("plugin") && f.EndsWith("dll"))
+                {
+                    var name = Path.GetFileNameWithoutExtension(file);
+                    try
+                    {
+                        Assembly.Load(name);
+                    }
+                    catch (Exception ex)
+                    {
+                        throw new Exception($"Failed to load plugin from file '{name}'.", ex);
+                    }
+                }
+            }
+        }
+    }
+}
@@ -0,0 +1,78 @@
+namespace Core
+{
+    internal class PluginManager
+    {
+        private readonly List<PluginToolsPair> pairs = new List<PluginToolsPair>();
+
+        internal void InstantiatePlugins(Type[] pluginTypes, IToolsFactory provider)
+        {
+            pairs.Clear();
+            foreach (var pluginType in pluginTypes)
+            {
+                var tools = provider.CreateTools();
+                var plugin = InstantiatePlugins(pluginType, tools);
+
+                ApplyLogPrefix(plugin, tools);
+            }
+        }
+
+        internal void AnnouncePlugins()
+        {
+            foreach (var pair in pairs) pair.Plugin.Announce();
+        }
+
+        internal PluginMetadata GatherPluginMetadata()
+        {
+            var metadata = new PluginMetadata();
+            foreach (var pair in pairs)
+            {
+                if (pair.Plugin is IHasMetadata m)
+                {
+                    m.AddMetadata(metadata);
+                }
+            }
+            return metadata;
+        }
+
+        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
+        {
+            foreach (var pair in pairs)
+            {
+                pair.Plugin.Decommission();
+                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
+            }
+        }
+
+        internal T GetPlugin<T>() where T : IProjectPlugin
+        {
+            return (T)pairs.Single(p => p.Plugin.GetType() == typeof(T)).Plugin;
+        }
+
+        private IProjectPlugin InstantiatePlugins(Type pluginType, PluginTools tools)
+        {
+            var plugin = (IProjectPlugin)Activator.CreateInstance(pluginType, args: tools)!;
+            pairs.Add(new PluginToolsPair(plugin, tools));
+            return plugin;
+        }
+
+        private void ApplyLogPrefix(IProjectPlugin plugin, PluginTools tools)
+        {
+            if (plugin is IHasLogPrefix hasLogPrefix)
+            {
+                tools.ApplyLogPrefix(hasLogPrefix.LogPrefix);
+            }
+        }
+
+        private class PluginToolsPair
+        {
+            public PluginToolsPair(IProjectPlugin plugin, IPluginTools tools)
+            {
+                Plugin = plugin;
+                Tools = tools;
+            }
+
+            public IProjectPlugin Plugin { get; }
+            public IPluginTools Tools { get; }
+        }
+    }
+}
@@ -0,0 +1,27 @@
+namespace Core
+{
+    internal interface IPluginMetadata
+    {
+        Dictionary<string, string> Get();
+    }
+
+    public interface IAddMetadata
+    {
+        void Add(string key, string value);
+    }
+
+    internal class PluginMetadata : IPluginMetadata, IAddMetadata
+    {
+        private readonly Dictionary<string, string> metadata = new Dictionary<string, string>();
+
+        public void Add(string key, string value)
+        {
+            metadata.Add(key, value);
+        }
+
+        public Dictionary<string, string> Get()
+        {
+            return new Dictionary<string, string>(metadata);
+        }
+    }
+}
@@ -0,0 +1,97 @@
+using FileUtils;
+using KubernetesWorkflow;
+using Logging;
+
+namespace Core
+{
+    public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
+    {
+        ITimeSet TimeSet { get; }
+
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone);
+    }
+
+    public interface IWorkflowTool
+    {
+        IStartupWorkflow CreateWorkflow(string? namespaceOverride = null);
+    }
+
+    public interface ILogTool
+    {
+        ILog GetLog();
+    }
+
+    public interface IHttpFactoryTool
+    {
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated);
+        IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet timeSet);
+        IHttp CreateHttp(string id);
+    }
+
+    public interface IFileTool
+    {
+        IFileManager GetFileManager();
+    }
+
+    internal class PluginTools : IPluginTools
+    {
+        private readonly WorkflowCreator workflowCreator;
+        private readonly IFileManager fileManager;
+        private readonly LogPrefixer log;
+
+        internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet)
+        {
+            this.log = new LogPrefixer(log);
+            this.workflowCreator = workflowCreator;
+            TimeSet = timeSet;
+            fileManager = new FileManager(log, fileManagerRootFolder);
+        }
+
+        public ITimeSet TimeSet { get; }
+
+        public void ApplyLogPrefix(string prefix)
+        {
+            log.Prefix = prefix;
+        }
+
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated)
+        {
+            return CreateHttp(id, onClientCreated, TimeSet);
+        }
+
+        public IHttp CreateHttp(string id, Action<HttpClient> onClientCreated, ITimeSet ts)
+        {
+            return new Http(id, log, ts, onClientCreated);
+        }
+
+        public IHttp CreateHttp(string id)
+        {
+            return new Http(id, log, TimeSet);
+        }
+
+        public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)
+        {
+            return workflowCreator.CreateWorkflow(namespaceOverride);
+        }
+
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
+        {
+            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace(waitTillDone);
+            if (deleteTrackedFiles) fileManager.DeleteAllFiles();
+        }
+
+        public IFileManager GetFileManager()
+        {
+            return fileManager;
+        }
+
+        public ILog GetLog()
+        {
+            return log;
+        }
+    }
+}
@@ -0,0 +1,34 @@
+using Utils;
+
+namespace Core
+{
+    public interface IProjectPlugin
+    {
+        void Announce();
+        void Decommission();
+    }
+
+    public interface IHasLogPrefix
+    {
+        string LogPrefix { get; }
+    }
+
+    public interface IHasMetadata
+    {
+        void AddMetadata(IAddMetadata metadata);
+    }
+
+    public static class ProjectPlugin
+    {
+        /// <summary>
+        /// On some platforms and in some cases, not all required plugin assemblies are automatically loaded into the app domain.
+        /// In this case, the runtime needs a slight push to load it before the EntryPoint class is instantiated.
+        /// Used ProjectPlugin.Load<>() before you create an EntryPoint to ensure all plugins you want to use are loaded.
+        /// </summary>
+        public static void Load<T>() where T : IProjectPlugin
+        {
+            var type = typeof(T);
+            FrameworkAssert.That(type != null, $"Unable to load plugin.");
+        }
+    }
+}
@@ -0,0 +1,18 @@
+using Newtonsoft.Json;
+
+namespace Core
+{
+    public static class SerializeGate
+    {
+        /// <summary>
+        /// SerializeGate was added to help ensure deployment objects are serializable and remain viable after deserialization.
+        /// Tools can be built on top of the core interface that rely on deployment objects being serializable.
+        /// Insert the serialization gate after deployment but before wrapping to ensure any future changes don't break this requirement.
+        /// </summary>
+        public static T Gate<T>(T anything)
+        {
+            var json = JsonConvert.SerializeObject(anything);
+            return JsonConvert.DeserializeObject<T>(json)!;
+        }
+    }
+}
@@ -0,0 +1,88 @@
+namespace Core
+{
+    public interface ITimeSet
+    {
+        /// <summary>
+        /// Timeout for a single HTTP call.
+        /// </summary>
+        TimeSpan HttpCallTimeout();
+
+        /// <summary>
+        /// Maximum total time to attempt to make a successful HTTP call to a service.
+        /// When HTTP calls time out during this timespan, retries will be made.
+        /// </summary>
+        TimeSpan HttpRetryTimeout();
+
+        /// <summary>
+        /// After a failed HTTP call, wait this long before trying again.
+        /// </summary>
+        TimeSpan HttpCallRetryDelay();
+
+        /// <summary>
+        /// After a failed K8s operation, wait this long before trying again.
+        /// </summary>
+        TimeSpan K8sOperationRetryDelay();
+
+        /// <summary>
+        /// Maximum total time to attempt to perform a successful k8s operation.
+        /// If k8s operations fail during this timespan, retries will be made.
+        /// </summary>
+        TimeSpan K8sOperationTimeout();
+    }
+
+    public class DefaultTimeSet : ITimeSet
+    {
+        public TimeSpan HttpCallTimeout()
+        {
+            return TimeSpan.FromMinutes(2);
+        }
+
+        public TimeSpan HttpRetryTimeout()
+        {
+            return TimeSpan.FromMinutes(5);
+        }
+
+        public TimeSpan HttpCallRetryDelay()
+        {
+            return TimeSpan.FromSeconds(1);
+        }
+
+        public TimeSpan K8sOperationRetryDelay()
+        {
+            return TimeSpan.FromSeconds(10);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromMinutes(30);
+        }
+    }
+
+    public class LongTimeSet : ITimeSet
+    {
+        public TimeSpan HttpCallTimeout()
+        {
+            return TimeSpan.FromMinutes(30);
+        }
+
+        public TimeSpan HttpRetryTimeout()
+        {
+            return TimeSpan.FromHours(2.2);
+        }
+
+        public TimeSpan HttpCallRetryDelay()
+        {
+            return TimeSpan.FromSeconds(20);
+        }
+
+        public TimeSpan K8sOperationRetryDelay()
+        {
+            return TimeSpan.FromSeconds(30);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromHours(1);
+        }
+    }
+}
@@ -0,0 +1,31 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace Core
+{
+    internal interface IToolsFactory
+    {
+        PluginTools CreateTools();
+    }
+
+    internal class ToolsFactory : IToolsFactory
+    {
+        private readonly ILog log;
+        private readonly WorkflowCreator workflowCreator;
+        private readonly string fileManagerRootFolder;
+        private readonly ITimeSet timeSet;
+
+        public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        {
+            this.log = log;
+            workflowCreator = new WorkflowCreator(log, configuration);
+            this.fileManagerRootFolder = fileManagerRootFolder;
+            this.timeSet = timeSet;
+        }
+
+        public PluginTools CreateTools()
+        {
+            return new PluginTools(log, workflowCreator, fileManagerRootFolder, timeSet);
+        }
+    }
+}
@@ -0,0 +1,21 @@
+using Utils;
+
+namespace DiscordRewards
+{
+    public class CheckConfig
+    {
+        public CheckType Type { get; set; }
+        public ulong MinNumberOfHosts { get; set; }
+        public ByteSize MinSlotSize { get; set; } = 0.Bytes();
+        public TimeSpan MinDuration { get; set; } = TimeSpan.Zero;
+    }
+
+    public enum CheckType
+    {
+        Uninitialized,
+        HostFilledSlot,
+        HostFinishedSlot,
+        ClientPostedContract,
+        ClientStartedContract,
+    }
+}
@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,19 @@
+namespace DiscordRewards
+{
+    public class GiveRewardsCommand
+    {
+        public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
+        public string[] EventsOverview { get; set; } = Array.Empty<string>();
+
+        public bool HasAny()
+        {
+            return Rewards.Any() || EventsOverview.Any();
+        }
+    }
+
+    public class RewardUsersCommand
+    {
+        public ulong RewardId { get; set; }
+        public string[] UserAddresses { get; set; } = Array.Empty<string>();
+    }
+}
@@ -0,0 +1,18 @@
+namespace DiscordRewards
+{
+    public class RewardConfig
+    {
+        public const string UsernameTag = "<USER>";
+
+        public RewardConfig(ulong roleId, string message, CheckConfig checkConfig)
+        {
+            RoleId = roleId;
+            Message = message;
+            CheckConfig = checkConfig;
+        }
+
+        public ulong RoleId { get; }
+        public string Message { get; }
+        public CheckConfig CheckConfig { get; }
+    }
+}
@@ -0,0 +1,53 @@
+namespace DiscordRewards
+{
+    public class RewardRepo
+    {
+        private static string Tag => RewardConfig.UsernameTag;
+
+        public RewardConfig[] Rewards { get; } = new RewardConfig[0];
+
+        // Example configuration, from test server:
+        //{
+        //    // Filled any slot
+        //    new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFilledSlot
+        //    }),
+
+        //    // Finished any slot
+        //    new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFinishedSlot
+        //    }),
+
+        //    // Finished a sizable slot
+        //    new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFinishedSlot,
+        //        MinSlotSize = 10.MB(),
+        //        MinDuration = TimeSpan.FromMinutes(5.0),
+        //    }),
+
+        //    // Posted any contract
+        //    new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientPostedContract
+        //    }),
+
+        //    // Started any contract
+        //    new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientStartedContract
+        //    }),
+
+        //    // Started a sizable contract
+        //    new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientStartedContract,
+        //        MinNumberOfHosts = 4,
+        //        MinSlotSize = 10.MB(),
+        //        MinDuration = TimeSpan.FromMinutes(5.0),
+        //    })
+        //};
+    }
+}
@@ -0,0 +1,162 @@
+using Logging;
+using Utils;
+
+namespace FileUtils
+{
+    public interface IFileManager
+    {
+        TrackedFile CreateEmptyFile(string label = "");
+        TrackedFile GenerateFile(ByteSize size, string label = "");
+        TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "");
+        void DeleteAllFiles();
+        void ScopedFiles(Action action);
+        T ScopedFiles<T>(Func<T> action);
+    }
+
+    public interface IGenerateOption
+    {
+        IGenerateOption Random(ByteSize size);
+        IGenerateOption StringRepeat(string str, ByteSize size);
+        IGenerateOption StringRepeat(string str, int times);
+        IGenerateOption ByteRepeat(byte[] bytes, ByteSize size);
+        IGenerateOption ByteRepeat(byte[] bytes, int times);
+    }
+
+    public class FileManager : IFileManager
+    {
+        private static readonly NumberSource folderNumberSource = new NumberSource(0);
+        private readonly ILog log;
+        private readonly string folder;
+        private readonly List<List<TrackedFile>> fileSetStack = new List<List<TrackedFile>>();
+
+        public const int ChunkSize = 1024 * 1024 * 100;
+
+        public FileManager(ILog log, string rootFolder)
+        {
+            folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
+
+            this.log = log;
+        }
+
+        public TrackedFile CreateEmptyFile(string label = "")
+        {
+            var path = Path.Combine(folder, Guid.NewGuid().ToString() + ".bin");
+            EnsureDirectory();
+
+            var result = new TrackedFile(log, path, label);
+            File.Create(result.Filename).Close();
+            if (fileSetStack.Any()) fileSetStack.Last().Add(result);
+            return result;
+        }
+
+        public TrackedFile GenerateFile(ByteSize size, string label = "")
+        {
+            return GenerateFile(o => o.Random(size), label);
+        }
+
+        public TrackedFile GenerateFile(Action<IGenerateOption> options, string label = "")
+        {
+            var sw = Stopwatch.Begin(log);
+            var result = RunGenerators(options, label);
+            sw.End($"Generated file {result.Describe()}.");
+            return result;
+        }
+
+        public void DeleteAllFiles()
+        {
+            DeleteDirectory();
+        }
+
+        public void ScopedFiles(Action action)
+        {
+            PushFileSet();
+            try
+            {
+                action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
+        }
+
+        public T ScopedFiles<T>(Func<T> action)
+        {
+            PushFileSet();
+            try
+            {
+                return action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
+        }
+
+        private void PushFileSet()
+        {
+            fileSetStack.Add(new List<TrackedFile>());
+        }
+
+        private void PopFileSet()
+        {
+            if (!fileSetStack.Any()) return;
+            var pop = fileSetStack.Last();
+            fileSetStack.Remove(pop);
+
+            foreach (var file in pop)
+            {
+                File.Delete(file.Filename);
+            }
+
+            // If the folder is now empty, delete it too.
+            if (!Directory.GetFiles(folder).Any()) DeleteDirectory();
+        }
+
+        private TrackedFile RunGenerators(Action<IGenerateOption> options, string label)
+        {
+            var result = CreateEmptyFile(label);
+            var generators = GetGenerators(options);
+            CheckSpaceAvailable(result, generators.GetRequiredSpace());
+
+            using var stream = new FileStream(result.Filename, FileMode.Append);
+            generators.Run(stream);
+            return result;
+        }
+
+        private GeneratorCollection GetGenerators(Action<IGenerateOption> options)
+        {
+            var result = new GeneratorCollection();
+            options(result);
+            return result;
+        }
+
+        private void CheckSpaceAvailable(TrackedFile testFile, long requiredSize)
+        {
+            var file = new FileInfo(testFile.Filename);
+            var drive = new DriveInfo(file.Directory!.Root.FullName);
+
+            var spaceAvailable = drive.TotalFreeSpace;
+
+            if (spaceAvailable < requiredSize)
+            {
+                var msg = $"Not enough disk space. " +
+                    $"{Formatter.FormatByteSize(requiredSize)} required. " +
+                    $"{Formatter.FormatByteSize(spaceAvailable)} available.";
+
+                log.Log(msg);
+                throw new Exception(msg);
+            }
+        }
+
+        private void EnsureDirectory()
+        {
+            Directory.CreateDirectory(folder);
+        }
+
+        private void DeleteDirectory()
+        {
+            if (Directory.Exists(folder)) Directory.Delete(folder, true);
+        }
+    }
+}
@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+    <ProjectReference Include="..\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>
@@ -0,0 +1,141 @@
+using System.Text;
+using Utils;
+
+namespace FileUtils
+{
+    public class GeneratorCollection : IGenerateOption
+    {
+        private readonly List<IGenerator> generators = new List<IGenerator>();
+
+        public IGenerateOption ByteRepeat(byte[] bytes, ByteSize size)
+        {
+            var times = size.SizeInBytes / bytes.Length;
+            generators.Add(new ByteRepeater(bytes, times));
+            return this;
+        }
+
+        public IGenerateOption ByteRepeat(byte[] bytes, int times)
+        {
+            generators.Add(new ByteRepeater(bytes, times));
+            return this;
+        }
+
+        public IGenerateOption Random(ByteSize size)
+        {
+            generators.Add(new RandomGenerator(size));
+            return this;
+        }
+
+        public IGenerateOption StringRepeat(string str, ByteSize size)
+        {
+            var times = size.SizeInBytes / str.Length;
+            generators.Add(new StringRepeater(str, times));
+            return this;
+        }
+
+        public IGenerateOption StringRepeat(string str, int times)
+        {
+            generators.Add(new StringRepeater(str, times));
+            return this;
+        }
+
+        public void Run(FileStream file)
+        {
+            foreach (var generator in generators)
+            {
+                generator.Generate(file);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return generators.Sum(g => g.GetRequiredSpace());
+        }
+    }
+
+    public interface IGenerator
+    {
+        void Generate(FileStream file);
+        long GetRequiredSpace();
+    }
+
+    public class ByteRepeater : IGenerator
+    {
+        private readonly byte[] bytes;
+        private readonly long times;
+
+        public ByteRepeater(byte[] bytes, long times)
+        {
+            this.bytes = bytes;
+            this.times = times;
+        }
+
+        public void Generate(FileStream file)
+        {
+            for (var i = 0; i < times; i++)
+            {
+                file.Write(bytes, 0, bytes.Length);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return bytes.Length * times;
+        }
+    }
+
+    public class StringRepeater : IGenerator
+    {
+        private readonly string str;
+        private readonly long times;
+
+        public StringRepeater(string str, long times)
+        {
+            this.str = str;
+            this.times = times;
+        }
+
+        public void Generate(FileStream file)
+        {
+            using var writer = new StreamWriter(file);
+            for (var i = 0; i < times; i++)
+            {
+                writer.Write(str);
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return Encoding.ASCII.GetBytes(str).Length * times;
+        }
+    }
+
+    public class RandomGenerator : IGenerator
+    {
+        private readonly Random random = new Random();
+        private readonly ByteSize size;
+
+        public RandomGenerator(ByteSize size)
+        {
+            this.size = size;
+        }
+
+        public void Generate(FileStream file)
+        {
+            var bytesLeft = size.SizeInBytes;
+            while (bytesLeft > 0)
+            {
+                var size = Math.Min(bytesLeft, FileManager.ChunkSize);
+                var bytes = new byte[size];
+                random.NextBytes(bytes);
+                file.Write(bytes, 0, bytes.Length);
+                bytesLeft -= size;
+            }
+        }
+
+        public long GetRequiredSpace()
+        {
+            return size.SizeInBytes;
+        }
+    }
+}
@@ -0,0 +1,87 @@
+using Logging;
+using Utils;
+
+namespace FileUtils
+{
+    public class TrackedFile
+    {
+        private readonly ILog log;
+
+        public TrackedFile(ILog log, string filename, string label)
+        {
+            this.log = log;
+            Filename = filename;
+            Label = label;
+        }
+
+        public string Filename { get; }
+        public string Label { get; }
+
+        public void AssertIsEqual(TrackedFile? actual)
+        {
+            var sw = Stopwatch.Begin(log);
+            try
+            {
+                AssertEqual(actual);
+            }
+            finally
+            {
+                sw.End($"{nameof(TrackedFile)}.{nameof(AssertIsEqual)}");
+            }
+        }
+
+        public string Describe()
+        {
+            var sizePostfix = $" ({Formatter.FormatByteSize(GetFileSize())})";
+            if (!string.IsNullOrEmpty(Label)) return Label + sizePostfix;
+            return $"'{Filename}'{sizePostfix}";
+        }
+
+        public ByteSize GetFilesize()
+        {
+            return new ByteSize(GetFileSize());
+        }
+
+        private void AssertEqual(TrackedFile? actual)
+        {
+            if (actual == null)  FrameworkAssert.Fail("TestFile is null.");
+            if (actual == this || actual!.Filename == Filename) FrameworkAssert.Fail("TestFile is compared to itself.");
+
+            FrameworkAssert.That(actual.GetFileSize() == GetFileSize(), "Files are not of equal length.");
+
+            using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
+            using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
+
+            var bytesExpected = new byte[FileManager.ChunkSize];
+            var bytesActual = new byte[FileManager.ChunkSize];
+
+            var readExpected = 0;
+            var readActual = 0;
+
+            while (true)
+            {
+                readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
+                readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
+
+                if (readExpected == 0 && readActual == 0)
+                {
+                    log.Log($"OK: {Describe()} is equal to {actual.Describe()}.");
+                    return;
+                }
+
+                FrameworkAssert.That(readActual == readExpected, "Unable to read buffers of equal length.");
+
+                for (var i = 0; i < readActual; i++)
+                {
+                    if (bytesExpected[i] != bytesActual[i]) FrameworkAssert.Fail("File contents not equal.");
+                }
+            }
+        }
+
+        private long GetFileSize()
+        {
+            var info = new FileInfo(Filename);
+            return info.Length;
+        }
+    }
+}
--- a/Show More
+++ b/Show More