working as intended

# Conflicts:
#	cs-codex-dist-testing.sln

.github/workflows/docker-autoclient.yml

@@ -0,0 +1,26 @@
+name: Docker - AutoClient
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/AutoClient/**'
+      - '!Tools/AutoClient/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-autoclient.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/AutoClient/docker/Dockerfile
+      docker_repo: codexstorage/codex-autoclient
+    secrets: inherit
+

.github/workflows/docker-discordbot.yml

@@ -0,0 +1,28 @@
+name: Docker - Discord Bot
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/BiblioTech/**'
+      - '!Tools/BiblioTech/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-discordbot.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/BiblioTech/docker/Dockerfile
+      docker_repo: codexstorage/codex-discordbot
+    secrets: inherit
+

.github/workflows/docker-keymaker.yml

@@ -0,0 +1,27 @@
+name: Docker - KeyMaker
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/KeyMaker/**'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-KeyMaker.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/KeyMaker/docker/Dockerfile
+      docker_repo: codexstorage/codex-keymaker
+    secrets: inherit
+

.github/workflows/docker-reusable.yml

@@ -0,0 +1,178 @@
+name: Reusable - Docker
+
+
+on:
+  workflow_call:
+    inputs:
+      docker_file:
+        default: docker/Dockerfile
+        description: Dockerfile
+        required: false
+        type: string
+      docker_repo:
+        default: codexstorage/cs-codex-dist-tests
+        description: DockerHub repository
+        required: false
+        type: string
+      tag_latest:
+        default: true
+        description: Set latest tag for Docker images
+        required: false
+        type: boolean
+      tag_sha:
+        default: true
+        description: Set Git short commit as Docker tag
+        required: false
+        type: boolean
+      tag_suffix:
+        default: ''
+        description: Suffix for Docker images tag
+        required: false
+        type: string
+
+
+env:
+  DOCKER_FILE: ${{ inputs.docker_file }}
+  DOCKER_REPO: ${{ inputs.docker_repo }}
+  TAG_LATEST: ${{ inputs.tag_latest }}
+  TAG_SHA: ${{ inputs.tag_sha }}
+  TAG_SUFFIX: ${{ inputs.tag_suffix }}
+
+
+jobs:
+  # Build platform specific image
+  build:
+    strategy:
+      fail-fast: true
+      matrix:
+        target:
+          - os: linux
+            arch: amd64
+          - os: linux
+            arch: arm64
+        include:
+          - target:
+              os: linux
+              arch: amd64
+            builder: ubuntu-22.04
+          - target:
+              os: linux
+              arch: arm64
+            builder: buildjet-4vcpu-ubuntu-2204-arm
+
+    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
+    runs-on: ${{ matrix.builder }}
+    env:
+      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_REPO }}
+
+      - name: Docker - Set up Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Build and Push by digest
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ${{ env.DOCKER_FILE }}
+          platforms: ${{ env.PLATFORM }}
+          push: true
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Docker - Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Docker - Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ matrix.target.arch }}
+          path: /tmp/digests
+          if-no-files-found: error
+          retention-days: 1
+
+
+  # Publish multi-platform image
+  publish:
+    name: Publish multi-platform image
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Docker - Variables
+        run: |
+          # Adjust custom suffix when set and
+          if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
+            echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
+          fi
+          # Disable SHA tags on tagged release
+          if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
+            echo "TAG_SHA=false" >>$GITHUB_ENV
+          fi
+          # Handle latest and latest-custom using raw
+          if [[ ${{ env.TAG_SHA }} == "false" ]]; then
+            echo "TAG_LATEST=false" >>$GITHUB_ENV
+            echo "TAG_RAW=true" >>$GITHUB_ENV
+            if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
+              echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
+            else
+              echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
+            fi
+          else
+            echo "TAG_RAW=false" >>$GITHUB_ENV
+          fi
+
+      - name: Docker - Download digests
+        uses: actions/download-artifact@v4
+        with:
+          pattern: digests-*
+          merge-multiple: true
+          path: /tmp/digests
+
+      - name: Docker - Set up Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_REPO }}
+          flavor: |
+            latest=${{ env.TAG_LATEST }}
+            suffix=${{ env.TAG_SUFFIX }},onlatest=true
+          tags: |
+            type=semver,pattern={{version}}
+            type=raw,enable=${{ env.TAG_RAW }},value=latest
+            type=sha,enable=${{ env.TAG_SHA }}
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
+
+      - name: Docker - Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}

.github/workflows/docker-rewarder.yml

@@ -0,0 +1,26 @@
+name: Docker - Rewarder Bot
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/TestNetRewarder/**'
+      - '!Tools/TestNetRewarder/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-rewarder.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/TestNetRewarder/docker/Dockerfile
+      docker_repo: codexstorage/codex-rewarderbot
+    secrets: inherit
+

.github/workflows/docker-runner.yml

@@ -0,0 +1,22 @@
+name: Docker - Runner
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - docker/Dockerfile
+      - docker/docker-entrypoint.sh
+      - .github/workflows/docker-runner.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    secrets: inherit

.github/workflows/report-HoldMyBeerTest.yaml

@@ -0,0 +1,20 @@
+name: Report - HoldMyBeerTest
+
+
+on:
+  # schedule:
+  #   - cron: '30 1 */2 * *'
+  workflow_dispatch:
+
+jobs:
+  run_tests:
+    name: Call runner
+    uses: ./.github/workflows/run-continuous-tests.yaml
+    with:
+      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+      branch: master
+      nameprefix: c-tests-report-holdmybeertest-48h
+      tests_target_duration: 48h
+      tests_filter: HoldMyBeerTest
+      tests_cleanup: true
+    secrets: inherit

.github/workflows/report-PeersTest.yaml

@@ -0,0 +1,20 @@
+name: Report - PeersTest
+
+
+on:
+  # schedule:
+  #   - cron: '30 1 */2 * *'
+  workflow_dispatch:
+
+jobs:
+  run_tests:
+    name: Call runner
+    uses: ./.github/workflows/run-continuous-tests.yaml
+    with:
+      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+      branch: master
+      nameprefix: c-tests-report-peerstest-48h
+      tests_target_duration: 48h
+      tests_filter: PeersTest
+      tests_cleanup: true
+    secrets: inherit

.github/workflows/run-continuous-tests.yaml

@@ -0,0 +1,170 @@
+name: Run Continuous Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
+        required: false
+        type: string
+      nameprefix:
+        description: Resources prefix (c-tests)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (2d)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup
+        required: false
+        type: boolean
+        default: true
+  workflow_call:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
+        required: false
+        type: string
+      nameprefix:
+        description: Resources prefix (c-tests)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (2d)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup
+        required: false
+        type: boolean
+        default: true
+      workflow_source:
+        description: Workflow source
+        required: false
+        type: string
+        default: ''
+
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
+  NAMEPREFIX: c-tests
+  NAMESPACE: default
+  TESTS_TARGET_DURATION: 2d
+  TESTS_FILTER: ""
+  TESTS_CLEANUP: true
+  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.28.2
+
+
+jobs:
+  run_tests:
+    name: Run Continuous Tests ${{ inputs.tests_filter }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
+          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          echo "DEPLOYMENT_NAMESPACE=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}"
+          echo "Runner pod: `kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}'`"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+          echo "Tests runid: ${{ env.RUNID }}"
+          echo "Tests codexdockerimage: ${{ env.CODEXDOCKERIMAGE }}"
+          echo "Tests namespace: ${{ env.DEPLOYMENT_NAMESPACE }}"
+          echo "Tests duration: ${{ env.TESTS_TARGET_DURATION }}"
+          echo "Tests filter: ${{ env.TESTS_FILTER }}"
+          echo "Tests cleanup: ${{ env.TESTS_CLEANUP }}"
+          echo "----"
+
+      - name: Show Runner logs
+        if: false
+        run: |
+          # Variables
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          duration=600
+          namespace="${{ env.NAMESPACE }}"
+          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")
+
+          # Check Pod status
+          WAIT=120
+          SECONDS=0
+          sleep=1
+          while (( SECONDS < WAIT )); do
+            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath="{.status.phase}")
+            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in $phase state - Get the logs"; break; } || { echo "Pod $pod is in $phase state - Retry in $sleep seconds / $((WAIT - SECONDS))"; }
+            sleep $sleep
+          done
+
+          # Get logs
+          timeout $duration \
+            kubectl logs $pod \
+              -n $namespace \
+              -f \
+              --tail=-1 \
+              --timestamps || true

.github/workflows/run-dist-tests.yaml

@@ -0,0 +1,83 @@
+name: Run Dist Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      nameprefix:
+        description: Runner name prefix (d-tests-runner)
+        required: false
+        type: string
+      namespace:
+        description: Runner namespace (default)
+        required: false
+        type: string
+      command:
+        description: Command (dotnet test Tests/CodexTests)
+        required: false
+        type: string
+
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  NAMEPREFIX: d-tests-runner
+  NAMESPACE: default
+  COMMAND: dotnet test Tests/CodexTests
+  JOB_MANIFEST: docker/dist-tests-job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.28.2
+
+
+jobs:
+  run_tests:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Variables
+        run: |
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'$COMMAND'"')" >>"$GITHUB_ENV"
+          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+          echo "Tests runid: ${{ env.RUNID }}"
+          echo "Tests command: `jq -r '. | join(" ")' <<< '${{ env.COMMAND }}'`"
+          echo "----"

.gitignore

@@ -1,3 +1,5 @@
 .vs
 obj
 bin
+.vscode
+Tools/AutoClient/datapath

CONTRIBUTINGPLUGINS.MD

@@ -0,0 +1,158 @@
+# Distributed System Tests for Nim-Codex
+
+## Contributing plugins
+The testing framework was created for testing Codex. However, it's been designed such that other containerized projects can 'easily' be added.
+
+In this file, you'll see 'users' (in quote) mentioned once or twice. This refers to code/projects/tests which end up making use of your plugin. 'Users' come in many shapes and sizes and tend to have many differen use-cases in mind. Please consider this when reading this document and writing your plugin.
+
+## Checklist
+Your application must pass this checklist to be compatible with the framework:
+- It runs in a docker container.
+- It can be configured via environment variables. (You may need to create a docker image which contains a shell script, to pass some env-vars as CLI arguments to your application. Container command overrides do work, but are not equally reliable across container platforms. When in doubt: use env-var!)
+- It has network interaction:
+   - It exposes one or more APIs via one or more ports, OR
+   - It makes calls to other services. (OR both.)
+
+If your application's use-cases rely primarily on shell interaction, this framework might not be for you. The framework allows you to execute commands in containers AND read stdout/stderr responses. However, its focus during development has always been webservice API interactions.
+
+## Steps
+In order to add your project to the framework you must:
+1. Create a library assembly in the project plugins folder.
+1. It must contain a type that implements the `IProjectPlugin` interface from the `Core` assembly.
+1. If your plugin wants to expose any specific methods or objects to 'users', it must implement extensions for the `CoreInterface` type.
+1. If your plugin wants to run containers of its own project, it must provide a recipe.
+
+## Constructors & Tools
+Your implementation of `IProjectPlugin` must have a public constructor with a single argument of type `IPluginTools`, for example:
+```C#
+   public class MyPlugin : IProjectPlugin
+   {
+      public MyPlugin(IPluginTools tools)
+      {
+         ...
+      }
+
+      ...
+   }
+```
+
+## Plugin Interfaces
+The `IProjectPlugin` interface requires the implementation of two methods.
+1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version and/or configuration information at this time if applicable.
+1. `Decommission` - Should your plugin have any active system resources, free them in this method. Please note that resources managed by the framework (such as running containers and tracked data files) do *not* need to be manually disposed in this method. `Decommission` is to be used for resources not managed by the framework.
+
+There are a few optional interfaces your plugin may choose to implement. The framework will automatically use these interfaces.
+1. `IHasLogPrefix` - Implementing this interface allows you to provide a string which will be prepended to all log statements made by your plugin. A polite thing to do.
+1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by 'users' of your plugin. Often this data finds its way into log files and container-descriptors in order to help track versions/tests/deployments, etc.
+
+## IPluginTools
+`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients to make calls to containers. (Figure out addresses and ports for containers is handled by the framework.)
+
+It is possible and allowed for your plugin to depend on and use other plugins. (For example, maybe your project wants to interact with Ethereum and wants to use the GethPlugin to talk to a Geth node.) `IPluginTools` is *not* what is used for accessing functionality of other plugins. See 'Core Interface' section.
+
+ILog GetLog();
+IHttp CreateHttp(Action<HttpClient> onClientCreated);
+IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
+IHttp CreateHttp();
+IFileManager GetFileManager();
+
+The plugin tools provide:
+1. `Workflow` - This tool allows you to start and stop containers using "container recipes". (More on those below.) It also allows you to execute commands inside a container, access stdout/stderr, detect crashes, and access pod deployment information. The workflow tool also lets you inspect the locations available in the cluster, and decide where you want to run containers. (More on that below as well.)
+1. `Log` - Good logging is priceless. Use this tool to get a log object handle, and write useful debug/info/error statements.
+1. `Http` - This tool gives you a convenient way to access a standard dotnet HttpClient, and takes care of timeouts and retries (in accordance with the config). Additionally, it combos nicely with container objects created by `Workflow`, such that you never have to spend any time figuring out the addresses and ports of your containers.
+1. `FileManager` - Lets you use tracked temporary files. Even if the 'user' tests/application start crashing, the framework will make sure these are cleaned up.
+
+## Core Interface
+Any functionality your plugin wants to expose to 'users' will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type:
+```C#
+   public static class CoreInterfaceExtensions
+   {
+      public static MyPluginReturnType DoSomethingCool(this CoreInterface ci, string someArgument)
+      {
+         return Plugin(ci).SomethingCool(someArgument);
+      }
+
+      private static MyPlugin Plugin(CoreInterface ci)
+      {
+         return ci.GetPlugin<MyPlugin>();
+      }
+   }
+```
+
+If your plugin wants to access the functionality exposed by other plugins, then you can pass the argument `CoreInterface ci` to your plugin code in order to do so. (For example, if you want to start a Geth node, the Geth plugin adds `IGethNode StartGethNode(this CoreInterface ci, Action<IGethSetup> setup)` to the core interface.) Don't forget you'll need to add a project reference to each plugin project you wish to use.
+
+While technically you can build whatever you like on top of the `CoreInterface` and your own plugin types, I recommend that you follow the approach explained below.
+
+## Deploying, Wrapping, and Starting
+When building a plugin, it is important to make as few assumptions as possible about how it will be used by whoever is going to use the framework. For this reason, I recommend you expose three kinds of methods using your `CoreInterface` extensions:
+1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning container objects as a result. If your project requires additional information, you can create a new class type to contain both it and the container objects created.
+1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain. (For example, if my project has a REST API call that allows users to fetch some state information, the object returned by Wrap should have a convenient method to call that API and receive that state information.)
+1. Start - This kind of method does both, simply calling a Deploy method first, then a Wrap method, and returns the result.
+
+Here's an example:
+```C#
+public static class CoreInterfaceExtensions
+   {
+      public static RunningContainers DeployMyProject(this CoreInterface ci, string someArgument)
+      {
+         // `RunningContainers` is a framework type. It contains all necessary information about a deployed container. It is serializable.
+         // Should you need to return any additional information, create a new type that contains it as well as the container information. Make sure it is serializable.
+         return Plugin(ci).DeployMyProjectContainer(someArgument); // <-- This method should use the `PluginTools.CreateWorkflow()` tool to deploy a container with a configuration that matches someArguments.
+      }
+
+      public static IMyProjectNode WrapMyProjectContainer(this CoreInterface ci, RunningContainers container)
+      {
+         return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` to create an HTTP client for the container, then wrap it in an object that
+         // represents the API of your project, in this case 'IMyProjectNode'.
+      }
+
+      public static IMyProjectNode StartMyProject(this CoreInterface ci, string someArgument)
+      {
+         // Start is now nothing more than a convenience method, combining the previous two.
+         var rc = ci.DeployMyProject(someArgument);
+         return WrapMyProjectContainer(ci, rc);
+      }
+   }
+```
+
+Should your deploy methods not return framework-types like RunningContainers, please make sure that your custom times are serializable. (Decorate them with the `[Serializable]` attribute.) Tools have been built using this framework which rely on the ability to serialize and store deployment information for later use. Please don't break this possibility. (Consider using the `SerializeGate` type to help ensure compatibility.)
+
+The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinuousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing.
+
+## Container Recipes
+In order to run a container of your application, the framework needs to know how to create that container. Think of a container recipe as being similar to a docker-compose.yaml file: You specify the docker image, ports, environment variables, persistent volumes, and secrets. However, container recipes are code. This allows you to add conditional behaviour to how your container is constructed. For example: The 'user' of your plugin specifies in their call input that they want to run your application in a certain mode. This would cause your container recipe to set certain environment variables, which cause the application to behave in the requested way.
+
+### Addresses and ports
+In a docker-compose.yaml file, it is perfectly normal to specify which ports should be exposed on your container. However, in the framework there's more to consider. When your application container starts, who knows on what kind of machine it runs, and what other processes it's sharing space with? Well, Kubernetes knows. Therefore, it is recommended that container recipes *do not* specify exact port numbers. The framework allows container recipes to declare "a port" without specifying its port number. This allows the framework and Kubernetes to figure out which ports are available when it's time to deploy. In order to find out which port numbers were assigned post-deployment, you can look up the port by tag (which is just an identifying string). When you specify a port to be mapped in your container recipe, you must specify:
+1. `Tag` - An identifier.
+1. `Internal` or `External` - Whether this port should be accessible only inside the cluster (for other containers (k8s: "ClusterIP")) or outside the cluster as well (for external tools/applications (k8s: "NodePort")).
+1. `Protocol` - TCP or UDP. Both protocols on the same port is not universally supported by all container engines, and is therefore not supported by the framework.
+
+If your application wants to listen for incoming traffic from inside its container, be sure to bind it to address "0.0.0.0".
+
+Reminder: If you don't want to worry about addresses, and internal or external ports, you don't have to! The container objects returned by the `workflow` plugin tool have a method called `GetAddress`. Given a port tag, it returns and address object. The `Http` plugin tool can use that address object to set up connections.
+
+## Locations
+The framework is designed to allow you to control instances of your application in multiple (physical) locations. It accomplishes this by using kubernetes, and the ability to deploy containers to specific hosts (nodes) inside a kubernetes cluster. Since Kubernetes allows you to build clusters cross-site, this framework in theory enables you to deploy and interact with containers running anywhere.
+
+The `workflow` plugin tool provides you a list of all available locations in the cluster. When starting a container, you are able to pick one of those locations. If no location is selected, one will be chosen by kubernetes. Locations can be chosen explicitly by kubernetes node name, or, they can be picked from the array of available locations.
+
+Example:
+```C#
+{
+   var location = Ci.GetKnownLocations().Get("kbnode_euwest_paris1");
+   var codex = Ci.StartCodexNode(s => s.At(location));
+}
+```
+In this example, 'Ci' is an instance of the core interface. The CodexPlugin exposes a function 'StartCodexNode', which allows its user to specify a location. This location is then passed to the `workflow` tool when the Codex plugin starts its container.
+
+The available locations array guarantees that each entry corresponds to a different kubernetes host.
+```C#
+{
+   var knownLocations = Ci.GetKnownLocations();
+   // I don't care where exactly, as long as they are different locations.
+   var codexAtZero = Ci.StartCodexNode(s => s.At(knownLocations.Get(0)));
+   var codexAtOne = Ci.StartCodexNode(s => s.At(knownLocations.Get(1)));
+}
+```
+

CONTRIBUTINGTESTS.MD

@@ -0,0 +1,40 @@
+# Distributed System Tests for Nim-Codex
+
+## Contributing tests
+Do you want to write some tests using this distributed test setup? Great! Here's what you do.
+
+1. Create a branch. Name it something descriptive, but start it with `tests/` please. [Example: `tests/data-redundancy`.]
+1. Checkout your branch.
+1. Create a new assembly in the `/Tests` folder. This can be an NUnit test assembly or simply a console app.
+1. Add Project references to `Core`, as well as any project plugin you'll be using.
+1. Write tests! Use existing tests for inspiration.
+
+## Tips for writing tests for Codex
+### Transient tests
+1. Add new code files to `Tests/CodexTests`
+1. Inherrit from `CodexDistTest` or `AutoBootstrapDistTest`.
+1. When using `CodexDistTest`:
+   1. You must start your own Codex bootstrap node. You can use `AddCodex(...)` for this.
+   1. When you start other Codex nodes with `AddCodex(...)` you can pass the bootstrap node by adding the `.WithBootstrapNode(...)` option.
+1. When using `AutoBootstrapDistTest`:
+   1. The test-infra creates the bootstrap node for you, and automatically passes it to each Codex node you create in your tests. Handy for keeping your tests clean and to-the-point.
+   1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `CodexDistTest` instead.
+1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
+### Continuous tests
+1. Add new code files to `Tests/CodexContinuousTests/Tests`
+1. Inherrit from `ContinuousTest`
+1. Define one or more methods and decorate them with the `[TestMoment(...)]` attribute.
+1. The TestMoment takes a number of seconds as argument. Each moment will be executed by the continuous test runner applying the given seconds as delay. (Non-cumulative. So two moments at T:10 will be executed one after another without delay, in this case the order of execution should not be depended upon.)
+1. Continuous tests automatically receive access to the Codex nodes that the tests are being run against.
+1. Additionally, Continuous tests can start their own transient Codex nodes and bootstrap them against the persistent nodes.
+
+### Tips for either type of test
+1. You can generate files of random test data by calling `GenerateTestFile(...)`.
+1. You can enable access to the Codex node metrics by adding the option `.EnableMetrics()`. Enabling metrics will make the test-infra download and save all Codex metrics in case of a test failure. (The metrics are stored as CSV, in the same location as the test log file.)
+1. You can enable access to the blockchain marketplace by adding the option `.EnableMarketplace(...)`.
+1. Enabling metrics and/or enabling the marketplace takes extra resources from the test-infra and increases the time needed during Codex node setup. Please don't enable these features unless your tests need them.
+1. Tip: Codex nodes can be named. Use the option `WithName(...)` and make reading your test logs a little nicer!
+1. Tip: Commit often.
+
+## Don't forget
+1. Once you're happy with your tests, please create a pull-request and ask a Codex core contributor to review your changes.

DistTestCore/ByteSize.cs

@@ -1,57 +0,0 @@
-namespace DistTestCore
-{
-    public class ByteSize
-    {
-        public ByteSize(long sizeInBytes)
-        {
-            SizeInBytes = sizeInBytes;
-        }
-
-        public long SizeInBytes { get; }
-    }
-
-    public static class IntExtensions
-    {
-        private const long Kilo = 1024;
-
-        public static ByteSize KB(this long i)
-        {
-            return new ByteSize(i * Kilo);
-        }
-
-        public static ByteSize MB(this long i)
-        {
-            return (i * Kilo).KB();
-        }
-
-        public static ByteSize GB(this long i)
-        {
-            return (i * Kilo).MB();
-        }
-
-        public static ByteSize TB(this long i)
-        {
-            return (i * Kilo).GB();
-        }
-
-        public static ByteSize KB(this int i)
-        {
-            return Convert.ToInt64(i).KB();
-        }
-
-        public static ByteSize MB(this int i)
-        {
-            return Convert.ToInt64(i).MB();
-        }
-
-        public static ByteSize GB(this int i)
-        {
-            return Convert.ToInt64(i).GB();
-        }
-
-        public static ByteSize TB(this int i)
-        {
-            return Convert.ToInt64(i).TB();
-        }
-    }
-}

DistTestCore/Codex/CodexAccess.cs

@@ -1,56 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore.Codex
-{
-    public class CodexAccess
-    {
-        public CodexAccess(RunningContainer runningContainer)
-        {
-            Container = runningContainer;
-        }
-
-        public RunningContainer Container { get; }
-
-        public CodexDebugResponse GetDebugInfo()
-        {
-            return Http().HttpGetJson<CodexDebugResponse>("debug/info");
-        }
-
-        public string UploadFile(FileStream fileStream)
-        {
-            return Http().HttpPostStream("upload", fileStream);
-        }
-
-        public Stream DownloadFile(string contentId)
-        {
-            return Http().HttpGetStream("download/" + contentId);
-        }
-
-        private Http Http()
-        {
-            var ip = Container.Pod.Cluster.IP;
-            var port = Container.ServicePorts[0].Number;
-            return new Http(ip, port, baseUrl: "/api/codex/v1");
-        }
-
-        public string ConnectToPeer(string peerId, string peerMultiAddress)
-        {
-            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
-        }
-    }
-
-    public class CodexDebugResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string[] addrs { get; set; } = new string[0];
-        public string repo { get; set; } = string.Empty;
-        public string spr { get; set; } = string.Empty;
-        public CodexDebugVersionResponse codex { get; set; } = new();
-    }
-
-    public class CodexDebugVersionResponse
-    {
-        public string version { get; set; } = string.Empty;
-        public string revision { get; set; } = string.Empty;
-    }
-}

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -1,54 +0,0 @@
-using DistTestCore.Marketplace;
-using KubernetesWorkflow;
-
-namespace DistTestCore.Codex
-{
-    public class CodexContainerRecipe : ContainerRecipeFactory
-    {
-        public const string DockerImage = "thatbenbierens/nim-codex:sha-b204837";
-        public const string MetricsPortTag = "metrics_port";
-
-        protected override string Image => DockerImage;
-
-        protected override void Initialize(StartupConfig startupConfig)
-        {
-            var config = startupConfig.Get<CodexStartupConfig>();
-
-            AddExposedPortAndVar("API_PORT");
-            AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}");
-            AddInternalPortAndVar("DISC_PORT");
-
-            var listenPort = AddInternalPort();
-            AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
-
-            if (config.LogLevel != null)
-            {
-                AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
-            }
-            if (config.StorageQuota != null)
-            {
-                AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
-            }
-            if (config.MetricsEnabled)
-            {
-                AddEnvVar("METRICS_ADDR", "0.0.0.0");
-                AddInternalPortAndVar("METRICS_PORT", tag: MetricsPortTag);
-            }
-
-            if (config.MarketplaceConfig != null)
-            {
-                var gethConfig = startupConfig.Get<GethStartResult>();
-                var companionNode = gethConfig.CompanionNodes[Index];
-                Additional(companionNode);
-
-                // Bootstrap node access from within the cluster:
-                //var ip = gethConfig.BootstrapNode.RunningContainers.RunningPod.Ip;
-                //var port = gethConfig.BootstrapNode.RunningContainers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag);
-
-                //AddEnvVar("ETH_PROVIDER", "todo");
-                //AddEnvVar("ETH_ACCOUNT", companionNode.Account);
-                //AddEnvVar("ETH_DEPLOYMENT", "todo");
-            }
-        }
-    }
-}

DistTestCore/Codex/CodexStartupConfig.cs

@@ -1,16 +0,0 @@
-using DistTestCore.Marketplace;
-using KubernetesWorkflow;
-
-namespace DistTestCore.Codex
-{
-    public class CodexStartupConfig
-    {
-        public Location Location { get; set; }
-        public CodexLogLevel? LogLevel { get; set; }
-        public ByteSize? StorageQuota { get; set; }
-        public bool MetricsEnabled { get; set; }
-        public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
-
-        //public IOnlineCodexNode? BootstrapNode { get; private set; }
-    }
-}

DistTestCore/CodexNodeFactory.cs

@@ -1,32 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Marketplace;
-using DistTestCore.Metrics;
-
-namespace DistTestCore
-{
-    public interface ICodexNodeFactory
-    {
-        OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group);
-    }
-
-    public class CodexNodeFactory : ICodexNodeFactory
-    {
-        private readonly TestLifecycle lifecycle;
-        private readonly IMetricsAccessFactory metricsAccessFactory;
-        private readonly IMarketplaceAccessFactory marketplaceAccessFactory;
-
-        public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory, IMarketplaceAccessFactory marketplaceAccessFactory)
-        {
-            this.lifecycle = lifecycle;
-            this.metricsAccessFactory = metricsAccessFactory;
-            this.marketplaceAccessFactory = marketplaceAccessFactory;
-        }
-
-        public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
-        {
-            var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container);
-            var marketplaceAccess = marketplaceAccessFactory.CreateMarketplaceAccess(access);
-            return new OnlineCodexNode(lifecycle, access, group, metricsAccess, marketplaceAccess);
-        }
-    }
-}

DistTestCore/CodexNodeGroup.cs

@@ -1,86 +0,0 @@
-using DistTestCore.Codex;
-using KubernetesWorkflow;
-using System.Collections;
-
-namespace DistTestCore
-{
-    public interface ICodexNodeGroup : IEnumerable<IOnlineCodexNode>
-    {
-        ICodexSetup BringOffline();
-        IOnlineCodexNode this[int index] { get; }
-    }
-
-    public class CodexNodeGroup : ICodexNodeGroup
-    {
-        private readonly TestLifecycle lifecycle;
-
-        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory)
-        {
-            this.lifecycle = lifecycle;
-            Setup = setup;
-            Containers = containers;
-            Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
-        }
-
-        public IOnlineCodexNode this[int index]
-        {
-            get
-            {
-                return Nodes[index];
-            }
-        }
-
-        public ICodexSetup BringOffline()
-        {
-            lifecycle.CodexStarter.BringOffline(this);
-
-            var result = Setup;
-            // Clear everything. Prevent accidental use.
-            Setup = null!;
-            Nodes = Array.Empty<OnlineCodexNode>();
-            Containers = null!;
-
-            return result;
-        }
-
-        public CodexSetup Setup { get; private set; }
-        public RunningContainers Containers { get; private set; }
-        public OnlineCodexNode[] Nodes { get; private set; }
-
-        public IEnumerator<IOnlineCodexNode> GetEnumerator()
-        {
-            return Nodes.Cast<IOnlineCodexNode>().GetEnumerator();
-        }
-
-        IEnumerator IEnumerable.GetEnumerator()
-        {
-            return Nodes.GetEnumerator();
-        }
-
-        public string Describe()
-        {
-            return $"<CodexNodeGroup@{Containers.Describe()}-{Setup.Describe()}>";
-        }
-
-        private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
-        {
-            var access = new CodexAccess(c);
-            EnsureOnline(access);
-            return factory.CreateOnlineCodexNode(access, this);
-        }
-
-        private void EnsureOnline(CodexAccess access)
-        {
-            try
-            {
-                var debugInfo = access.GetDebugInfo();
-                if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
-            }
-            catch (Exception e)
-            {
-                lifecycle.Log.Error($"Failed to start codex node: {e}");
-                throw;
-            }
-        }
-    }
-}

DistTestCore/CodexSetup.cs

@@ -1,84 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Marketplace;
-using KubernetesWorkflow;
-
-namespace DistTestCore
-{
-    public interface ICodexSetup
-    {
-        ICodexSetup At(Location location);
-        ICodexSetup WithLogLevel(CodexLogLevel level);
-        //ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node);
-        ICodexSetup WithStorageQuota(ByteSize storageQuota);
-        ICodexSetup EnableMetrics();
-        ICodexSetup EnableMarketplace(int initialBalance);
-        ICodexNodeGroup BringOnline();
-    }
-    
-    public class CodexSetup : CodexStartupConfig, ICodexSetup
-    {
-        private readonly CodexStarter starter;
-
-        public int NumberOfNodes { get; }
-
-        public CodexSetup(CodexStarter starter, int numberOfNodes)
-        {
-            this.starter = starter;
-            NumberOfNodes = numberOfNodes;
-        }
-
-        public ICodexNodeGroup BringOnline()
-        {
-            return starter.BringOnline(this);
-        }
-
-        public ICodexSetup At(Location location)
-        {
-            Location = location;
-            return this;
-        }
-
-        //public ICodexSetupConfig WithBootstrapNode(IOnlineCodexNode node)
-        //{
-        //    BootstrapNode = node;
-        //    return this;
-        //}
-
-        public ICodexSetup WithLogLevel(CodexLogLevel level)
-        {
-            LogLevel = level;
-            return this;
-        }
-
-        public ICodexSetup WithStorageQuota(ByteSize storageQuota)
-        {
-            StorageQuota = storageQuota;
-            return this;
-        }
-
-        public ICodexSetup EnableMetrics()
-        {
-            MetricsEnabled = true;
-            return this;
-        }
-
-        public ICodexSetup EnableMarketplace(int initialBalance)
-        {
-            MarketplaceConfig = new MarketplaceInitialConfig(initialBalance);
-            return this;
-        }
-
-        public string Describe()
-        {
-            var args = string.Join(',', DescribeArgs());
-            return $"({NumberOfNodes} CodexNodes with [{args}])";
-        }
-
-        private IEnumerable<string> DescribeArgs()
-        {
-            if (LogLevel != null) yield return $"LogLevel={LogLevel}";
-            //if (BootstrapNode != null) yield return "BootstrapNode=set-not-shown-here";
-            if (StorageQuota != null) yield return $"StorageQuote={StorageQuota.SizeInBytes}";
-        }
-    }
-}

DistTestCore/CodexStarter.cs

@@ -1,85 +0,0 @@
-using DistTestCore.Codex;
-using KubernetesWorkflow;
-
-namespace DistTestCore
-{
-    public class CodexStarter
-    {
-        private readonly TestLifecycle lifecycle;
-        private readonly WorkflowCreator workflowCreator;
-
-        public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-        {
-            this.lifecycle = lifecycle;
-            this.workflowCreator = workflowCreator;
-        }
-
-        public List<CodexNodeGroup> RunningGroups { get; } = new List<CodexNodeGroup>();
-
-        public ICodexNodeGroup BringOnline(CodexSetup codexSetup)
-        {
-            Log($"Starting {codexSetup.Describe()}...");
-            var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
-
-            var startupConfig = new StartupConfig();
-            startupConfig.Add(codexSetup);
-            startupConfig.Add(gethStartResult);
-
-            var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
-
-            var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers);
-            
-            var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
-
-            var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
-            Log($"Started at '{group.Containers.RunningPod.Ip}'");
-            return group;
-        }
-
-        public void BringOffline(CodexNodeGroup group)
-        {
-            Log($"Stopping {group.Describe()}...");
-            var workflow = CreateWorkflow();
-            workflow.Stop(group.Containers);
-            RunningGroups.Remove(group);
-            Log("Stopped.");
-        }
-
-        public void DeleteAllResources()
-        {
-            var workflow = CreateWorkflow();
-            workflow.DeleteAllResources();
-
-            RunningGroups.Clear();
-        }
-
-        public void DownloadLog(RunningContainer container, ILogHandler logHandler)
-        {
-            var workflow = CreateWorkflow();
-            workflow.DownloadContainerLog(container, logHandler);
-        }
-        
-        private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
-        {
-            var workflow = CreateWorkflow();
-            return workflow.Start(numberOfNodes, location, new CodexContainerRecipe(), startupConfig);
-        }
-
-        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory)
-        {
-            var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
-            RunningGroups.Add(group);
-            return group;
-        }
-
-        private StartupWorkflow CreateWorkflow()
-        {
-            return workflowCreator.CreateWorkflow();
-        }
-
-        private void Log(string msg)
-        {
-            lifecycle.Log.Log(msg);
-        }
-    }
-}

DistTestCore/Configuration.cs

@@ -1,32 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore
-{
-    public class Configuration
-    {
-        public KubernetesWorkflow.Configuration GetK8sConfiguration()
-        {
-            return new KubernetesWorkflow.Configuration(
-                k8sNamespace: "codex-test-ns",
-                kubeConfigFile: null,
-                operationTimeout: Timing.K8sOperationTimeout(),
-                retryDelay: Timing.K8sServiceDelay(),
-                locationMap: new[]
-                {
-                    new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),
-                    new ConfigurationLocationEntry(Location.BensLaptop, "worker02"),
-                } 
-            );
-        }
-
-        public Logging.LogConfig GetLogConfig()
-        {
-            return new Logging.LogConfig("D:/CodexTestLogs");
-        }
-
-        public string GetFileManagerFolder()
-        {
-            return "TestDataFiles";
-        }
-    }
-}

DistTestCore/DistTest.cs

@@ -1,194 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Logs;
-using DistTestCore.Marketplace;
-using DistTestCore.Metrics;
-using KubernetesWorkflow;
-using Logging;
-using NUnit.Framework;
-using Utils;
-
-namespace DistTestCore
-{
-    [SetUpFixture]
-    public abstract class DistTest
-    {
-        private readonly Configuration configuration = new Configuration();
-        private FixtureLog fixtureLog = null!;
-        private TestLifecycle lifecycle = null!;
-        private DateTime testStart = DateTime.MinValue;
-
-        [OneTimeSetUp]
-        public void GlobalSetup()
-        {
-            // Previous test run may have been interrupted.
-            // Begin by cleaning everything up.
-            fixtureLog = new FixtureLog(configuration.GetLogConfig());
-
-            try
-            {
-                Stopwatch.Measure(fixtureLog, "Global setup", () =>
-                {
-                    var wc = new WorkflowCreator(configuration.GetK8sConfiguration());
-                    wc.CreateWorkflow().DeleteAllResources();
-                });                
-            }
-            catch (Exception ex)
-            {
-                GlobalTestFailure.HasFailed = true;
-                Error($"Global setup cleanup failed with: {ex}");
-                throw;
-            }
-
-            fixtureLog.Log("Global setup cleanup successful");
-            fixtureLog.Log($"Codex image: {CodexContainerRecipe.DockerImage}");
-            fixtureLog.Log($"Prometheus image: {PrometheusContainerRecipe.DockerImage}");
-            fixtureLog.Log($"Geth image: {GethContainerRecipe.DockerImage}");
-        }
-
-        [SetUp]
-        public void SetUpDistTest()
-        {
-            if (GlobalTestFailure.HasFailed)
-            {
-                Assert.Inconclusive("Skip test: Previous test failed during clean up.");
-            }
-            else
-            {
-                CreateNewTestLifecycle();
-            }
-        }
-
-        [TearDown]
-        public void TearDownDistTest()
-        {
-            try
-            {
-                DisposeTestLifecycle();
-            }
-            catch (Exception ex)
-            {
-                Error("Cleanup failed: " + ex.Message);
-                GlobalTestFailure.HasFailed = true;
-            }
-        }
-
-        public TestFile GenerateTestFile(ByteSize size)
-        {
-            return lifecycle.FileManager.GenerateTestFile(size);
-        }
-
-        public ICodexSetup SetupCodexNodes(int numberOfNodes)
-        {
-            return new CodexSetup(lifecycle.CodexStarter, numberOfNodes);
-        }
-
-        private void IncludeLogsAndMetricsOnTestFailure()
-        {
-            var result = TestContext.CurrentContext.Result;
-            if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
-            {
-                fixtureLog.MarkAsFailed();
-
-                if (IsDownloadingLogsAndMetricsEnabled())
-                {
-                    Log("Downloading all CodexNode logs and metrics because of test failure...");
-                    DownloadAllLogs();
-                    DownloadAllMetrics();
-                }
-                else
-                {
-                    Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
-                }
-            }
-        }
-
-        private void Log(string msg)
-        {
-            lifecycle.Log.Log(msg);
-        }
-
-        private void Error(string msg)
-        {
-            lifecycle.Log.Error(msg);
-        }
-
-        private void CreateNewTestLifecycle()
-        {
-            Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () =>
-            {
-                lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration);
-                testStart = DateTime.UtcNow;
-            });
-        }
-
-        private void DisposeTestLifecycle()
-        {
-            fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({GetTestDuration()})");
-            Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
-            {
-                lifecycle.Log.EndTest();
-                IncludeLogsAndMetricsOnTestFailure();
-                lifecycle.DeleteAllResources();
-                lifecycle = null!;
-            });
-        }
-
-        private string GetTestDuration()
-        {
-            var testDuration = DateTime.UtcNow - testStart;
-            return Time.FormatDuration(testDuration);
-        }
-
-        private void DownloadAllLogs()
-        {
-            OnEachCodexNode(node =>
-            {
-                lifecycle.DownloadLog(node);
-            });
-        }
-
-        private void DownloadAllMetrics()
-        {
-            var metricsDownloader = new MetricsDownloader(lifecycle.Log);
-
-            OnEachCodexNode(node =>
-            {
-                var m = node.Metrics as MetricsAccess;
-                if (m != null)
-                {
-                    metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m);
-                }
-            });
-        }
-
-        private void OnEachCodexNode(Action<OnlineCodexNode> action)
-        {
-            var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
-            foreach (var node in allNodes)
-            {
-                action(node);
-            }
-        }
-
-        private string GetCurrentTestName()
-        {
-            return $"[{TestContext.CurrentContext.Test.Name}]";
-        }
-
-        private string GetTestResult()
-        {
-            return TestContext.CurrentContext.Result.Outcome.Status.ToString();
-        }
-
-        private bool IsDownloadingLogsAndMetricsEnabled()
-        {
-            var testProperties = TestContext.CurrentContext.Test.Properties;
-            return !testProperties.ContainsKey(DontDownloadLogsAndMetricsOnFailureAttribute.DontDownloadKey);
-        }
-    }
-
-    public static class GlobalTestFailure
-    {
-        public static bool HasFailed { get; set; } = false;
-    }
-}

DistTestCore/FileManager.cs

@@ -1,120 +0,0 @@
-using Logging;
-using NUnit.Framework;
-
-namespace DistTestCore
-{
-    public interface IFileManager
-    {
-        TestFile CreateEmptyTestFile();
-        TestFile GenerateTestFile(ByteSize size);
-        void DeleteAllTestFiles();
-    }
-
-    public class FileManager : IFileManager
-    {
-        public const int ChunkSize = 1024 * 1024;
-        private readonly Random random = new Random();
-        private readonly TestLog log;
-        private readonly string folder;
-
-        public FileManager(TestLog log, Configuration configuration)
-        {
-            folder = configuration.GetFileManagerFolder();
-
-            EnsureDirectory();
-            this.log = log;
-        }
-
-        public TestFile CreateEmptyTestFile()
-        {
-            var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin"));
-            File.Create(result.Filename).Close();
-            return result;
-        }
-
-        public TestFile GenerateTestFile(ByteSize size)
-        {
-            var result = CreateEmptyTestFile();
-            GenerateFileBytes(result, size);
-            log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'.");
-            return result;
-        }
-
-        public void DeleteAllTestFiles()
-        {
-            DeleteDirectory();
-        }
-
-        private void GenerateFileBytes(TestFile result, ByteSize size)
-        {
-            long bytesLeft = size.SizeInBytes;
-            while (bytesLeft > 0)
-            {
-                var length = Math.Min(bytesLeft, ChunkSize);
-                AppendRandomBytesToFile(result, length);
-                bytesLeft -= length;
-            }
-        }
-
-        private void AppendRandomBytesToFile(TestFile result, long length)
-        {
-            var bytes = new byte[length];
-            random.NextBytes(bytes);
-            using var stream = new FileStream(result.Filename, FileMode.Append);
-            stream.Write(bytes, 0, bytes.Length);
-        }
-
-        private void EnsureDirectory()
-        {
-            if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
-        }
-
-        private void DeleteDirectory()
-        {
-            Directory.Delete(folder, true);
-        }
-    }
-
-    public class TestFile
-    {
-        public TestFile(string filename)
-        {
-            Filename = filename;
-        }
-
-        public string Filename { get; }
-
-        public long GetFileSize()
-        {
-            var info = new FileInfo(Filename);
-            return info.Length;
-        }
-
-        public void AssertIsEqual(TestFile? actual)
-        {
-            if (actual == null) Assert.Fail("TestFile is null.");
-            if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
-
-            Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length.");
-
-            using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
-            using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
-
-            var bytesExpected = new byte[FileManager.ChunkSize];
-            var bytesActual = new byte[FileManager.ChunkSize];
-
-            var readExpected = 0;
-            var readActual = 0;
-
-            while (true)
-            {
-                readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
-                readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
-
-                if (readExpected == 0 && readActual == 0) return;
-                Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length.");
-                CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal.");
-            }
-        }
-    }
-}

DistTestCore/GethStarter.cs

@@ -1,80 +0,0 @@
-using DistTestCore.Marketplace;
-using KubernetesWorkflow;
-
-namespace DistTestCore
-{
-    public class GethStarter
-    {
-        private readonly GethBootstrapNodeCache bootstrapNodeCache;
-        private readonly GethCompanionNodeStarter companionNodeStarter;
-        private readonly TestLifecycle lifecycle;
-
-        public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-        {
-            bootstrapNodeCache = new GethBootstrapNodeCache(new GethBootstrapNodeStarter(lifecycle, workflowCreator));
-            companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator);
-            this.lifecycle = lifecycle;
-        }
-
-        public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup)
-        {
-            if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult();
-
-            var bootstrapNode = bootstrapNodeCache.Get();
-            var companionNodes = StartCompanionNodes(codexSetup, bootstrapNode);
-
-            TransferInitialBalance(bootstrapNode, codexSetup.MarketplaceConfig.InitialBalance, companionNodes);
-
-            return CreateGethStartResult(bootstrapNode, companionNodes);
-        }
-
-        private void TransferInitialBalance(GethBootstrapNodeInfo bootstrapNode, int initialBalance, GethCompanionNodeInfo[] companionNodes)
-        {
-            var interaction = bootstrapNode.StartInteraction(lifecycle.Log);
-            foreach (var node in companionNodes)
-            {
-                interaction.TransferTo(node.Account, initialBalance);
-            }
-        }
-
-        private GethStartResult CreateGethStartResult(GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo[] companionNodes)
-        {
-            return new GethStartResult(CreateMarketplaceAccessFactory(bootstrapNode), bootstrapNode, companionNodes);
-        }
-
-        private GethStartResult CreateMarketplaceUnavailableResult()
-        {
-            return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, Array.Empty<GethCompanionNodeInfo>());
-        }
-
-        private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(GethBootstrapNodeInfo bootstrapNode)
-        {
-            return new GethMarketplaceAccessFactory(lifecycle.Log, bootstrapNode!);
-        }
-
-        private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode)
-        {
-            return companionNodeStarter.StartCompanionNodesFor(codexSetup, bootstrapNode);
-        }
-    }
-
-    public class GethBootstrapNodeCache
-    {
-        private readonly GethBootstrapNodeStarter bootstrapNodeStarter;
-        private GethBootstrapNodeInfo? bootstrapNode;
-
-        public GethBootstrapNodeCache(GethBootstrapNodeStarter bootstrapNodeStarter)
-        {
-            this.bootstrapNodeStarter = bootstrapNodeStarter;
-        }
-
-        public GethBootstrapNodeInfo Get()
-        {
-            if (bootstrapNode == null)
-            {
-                bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode();
-            }
-            return bootstrapNode;
-        }
-    }
-}

DistTestCore/Http.cs

@@ -1,101 +0,0 @@
-using Newtonsoft.Json;
-using NUnit.Framework;
-using System.Net.Http.Headers;
-using Utils;
-
-namespace DistTestCore
-{
-    public class Http
-    {
-        private readonly string ip;
-        private readonly int port;
-        private readonly string baseUrl;
-
-        public Http(string ip, int port, string baseUrl)
-        {
-            this.ip = ip;
-            this.port = port;
-            this.baseUrl = baseUrl;
-
-            if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
-            if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
-        }
-
-        public string HttpGetString(string route)
-        {
-            return Retry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                var result = Time.Wait(client.GetAsync(url));
-                return Time.Wait(result.Content.ReadAsStringAsync());
-            });
-        }
-
-        public T HttpGetJson<T>(string route)
-        {
-            return JsonConvert.DeserializeObject<T>(HttpGetString(route))!;
-        }
-
-        public string HttpPostStream(string route, Stream stream)
-        {
-            return Retry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-
-                var content = new StreamContent(stream);
-                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
-                var response = Time.Wait(client.PostAsync(url, content));
-
-                return Time.Wait(response.Content.ReadAsStringAsync());
-            });
-        }
-
-        public Stream HttpGetStream(string route)
-        {
-            return Retry(() =>
-            {
-                var client = GetClient();
-                var url = GetUrl() + route;
-
-                return Time.Wait(client.GetStreamAsync(url));
-            });
-        }
-
-        private string GetUrl()
-        {
-            return $"http://{ip}:{port}{baseUrl}";
-        }
-
-        private static T Retry<T>(Func<T> operation)
-        {
-            var retryCounter = 0;
-
-            while (true)
-            {
-                try
-                {
-                    return operation();
-                }
-                catch (Exception exception)
-                {
-                    Timing.HttpCallRetryDelay();
-                    retryCounter++;
-                    if (retryCounter > Timing.HttpCallRetryCount())
-                    {
-                        Assert.Fail(exception.Message);
-                        throw;
-                    }
-                }
-            }
-        }
-
-        private static HttpClient GetClient()
-        {
-            var client = new HttpClient();
-            client.Timeout = Timing.HttpCallTimeout();
-            return client;
-        }
-    }
-}

DistTestCore/Logs/CodexNodeLog.cs

@@ -1,35 +0,0 @@
-using Logging;
-using NUnit.Framework;
-
-namespace DistTestCore.Logs
-{
-    public interface ICodexNodeLog
-    {
-        void AssertLogContains(string expectedString);
-    }
-
-    public class CodexNodeLog : ICodexNodeLog
-    {
-        private readonly LogFile logFile;
-
-        public CodexNodeLog(LogFile logFile)
-        {
-            this.logFile = logFile;
-        }
-
-        public void AssertLogContains(string expectedString)
-        {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (line.Contains(expectedString)) return;
-                line = streamReader.ReadLine();
-            }
-
-            Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
-        }
-    }
-}

DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs

@@ -1,15 +0,0 @@
-using NUnit.Framework;
-
-namespace DistTestCore.Logs
-{
-    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
-    public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute
-    {
-        public const string DontDownloadKey = "DontDownloadLogsAndMetrics";
-
-        public DontDownloadLogsAndMetricsOnFailureAttribute()
-            : base(DontDownloadKey)
-        {
-        }
-    }
-}

DistTestCore/Logs/LogDownloadHandler.cs

@@ -1,28 +0,0 @@
-using KubernetesWorkflow;
-using Logging;
-
-namespace DistTestCore.Logs
-{
-    public class LogDownloadHandler : LogHandler, ILogHandler
-    {
-        private readonly LogFile log;
-
-        public LogDownloadHandler(string description, LogFile log)
-        {
-            this.log = log;
-
-            log.Write($"{description} -->> {log.FullFilename}");
-            log.WriteRaw(description);
-        }
-
-        public CodexNodeLog CreateCodexNodeLog()
-        {
-            return new CodexNodeLog(log);
-        }
-
-        protected override void ProcessLine(string line)
-        {
-            log.WriteRaw(line);
-        }
-    }
-}

DistTestCore/Marketplace/GethBootstrapNodeInfo.cs

@@ -1,33 +0,0 @@
-using KubernetesWorkflow;
-using Logging;
-using NethereumWorkflow;
-
-namespace DistTestCore.Marketplace
-{
-    public class GethBootstrapNodeInfo
-    {
-        public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64, string pubKey, Port discoveryPort)
-        {
-            RunningContainers = runningContainers;
-            Account = account;
-            GenesisJsonBase64 = genesisJsonBase64;
-            PubKey = pubKey;
-            DiscoveryPort = discoveryPort;
-        }
-
-        public RunningContainers RunningContainers { get; }
-        public string Account { get; }
-        public string GenesisJsonBase64 { get; }
-        public string PubKey { get; }
-        public Port DiscoveryPort { get; }
-
-        public NethereumInteraction StartInteraction(TestLog log)
-        {
-            var ip = RunningContainers.RunningPod.Cluster.IP;
-            var port = RunningContainers.Containers[0].ServicePorts[0].Number;
-
-            var creator = new NethereumInteractionCreator(log, ip, port, Account);
-            return creator.CreateWorkflow();
-        }
-    }
-}

DistTestCore/Marketplace/GethBootstrapNodeStarter.cs

@@ -1,49 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore.Marketplace
-{
-    public class GethBootstrapNodeStarter
-    {
-        private const string bootstrapGenesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9";
-        private readonly TestLifecycle lifecycle;
-        private readonly WorkflowCreator workflowCreator;
-
-        public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-        {
-            this.lifecycle = lifecycle;
-            this.workflowCreator = workflowCreator;
-        }
-
-        public GethBootstrapNodeInfo StartGethBootstrapNode()
-        {
-            Log("Starting Geth bootstrap node...");
-            var startupConfig = CreateBootstrapStartupConfig();
-            
-            var workflow = workflowCreator.CreateWorkflow();
-            var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
-            if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
-
-            var extractor = new GethInfoExtractor(workflow, containers.Containers[0]);
-            var account = extractor.ExtractAccount();
-            var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64();
-            var pubKey = extractor.ExtractPubKey();
-            var discoveryPort = containers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
-
-            Log($"Geth bootstrap node started with account '{account}'");
-
-            return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64, pubKey, discoveryPort);
-        }
-
-        private StartupConfig CreateBootstrapStartupConfig()
-        {
-            var config = new StartupConfig();
-            config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64, null!));
-            return config;
-        }
-
-        private void Log(string msg)
-        {
-            lifecycle.Log.Log(msg);
-        }
-    }
-}

DistTestCore/Marketplace/GethCompanionNodeInfo.cs

@@ -1,16 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore.Marketplace
-{
-    public class GethCompanionNodeInfo
-    {
-        public GethCompanionNodeInfo(RunningContainer runningContainer, string account)
-        {
-            RunningContainer = runningContainer;
-            Account = account;
-        }
-
-        public RunningContainer RunningContainer { get; }
-        public string Account { get; }
-    }
-}

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -1,50 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore.Marketplace
-{
-    public class GethCompanionNodeStarter
-    {
-        private readonly TestLifecycle lifecycle;
-        private readonly WorkflowCreator workflowCreator;
-
-        public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-        {
-            this.lifecycle = lifecycle;
-            this.workflowCreator = workflowCreator;
-        }
-
-        public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode)
-        {
-            Log($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes.");
-
-            var startupConfig = CreateCompanionNodeStartupConfig(bootstrapNode);
-
-            var workflow = workflowCreator.CreateWorkflow();
-            var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig);
-            if (containers.Containers.Length != codexSetup.NumberOfNodes) throw new InvalidOperationException("Expected a Geth companion node to be created for each Codex node. Test infra failure.");
-
-            Log("Initialized companion nodes.");
-
-            return containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray();
-        }
-
-        private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container)
-        {
-            var extractor = new GethInfoExtractor(workflow, container);
-            var account = extractor.ExtractAccount();
-            return new GethCompanionNodeInfo(container, account);
-        }
-
-        private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode)
-        {
-            var config = new StartupConfig();
-            config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64, bootstrapNode));
-            return config;
-        }
-
-        private void Log(string msg)
-        {
-            lifecycle.Log.Log(msg);
-        }
-    }
-}

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -1,49 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore.Marketplace
-{
-    public class GethContainerRecipe : ContainerRecipeFactory
-    {
-        public const string DockerImage = "thatbenbierens/geth-confenv:latest";
-        public const string HttpPortTag = "http_port";
-        public const string DiscoveryPortTag = "disc_port";
-        public const string AccountFilename = "account_string.txt";
-        public const string GenesisFilename = "genesis.json";
-        
-        protected override string Image => DockerImage;
-
-        protected override void Initialize(StartupConfig startupConfig)
-        {
-            var config = startupConfig.Get<GethStartupConfig>();
-
-            var args = CreateArgs(config);
-
-            AddEnvVar("GETH_ARGS", args);
-            AddEnvVar("GENESIS_JSON", config.GenesisJsonBase64);
-        }
-
-        private string CreateArgs(GethStartupConfig config)
-        {
-            var discovery = AddInternalPort(tag: DiscoveryPortTag);
-
-            if (config.IsBootstrapNode)
-            {
-                AddEnvVar("IS_BOOTSTRAP", "1");
-                var exposedPort = AddExposedPort();
-                return $"--http.port {exposedPort.Number} --discovery.port {discovery.Number} --nodiscover";
-            }
-
-            var port = AddInternalPort();
-            var authRpc = AddInternalPort();
-            var httpPort = AddInternalPort(tag: HttpPortTag);
-
-            var bootPubKey = config.BootstrapNode.PubKey;
-            var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip;
-            var bootPort = config.BootstrapNode.DiscoveryPort.Number;
-            var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort}";
-            // geth --bootnodes enode://pubkey1@ip1:port1
-
-            return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number} --nodiscover {bootstrapArg}";
-        }
-    }
-}

DistTestCore/Marketplace/GethInfoExtractor.cs

@@ -1,110 +0,0 @@
-using KubernetesWorkflow;
-using System.Text;
-
-namespace DistTestCore.Marketplace
-{
-    public class GethInfoExtractor
-    {
-        private readonly StartupWorkflow workflow;
-        private readonly RunningContainer container;
-
-        public GethInfoExtractor(StartupWorkflow workflow, RunningContainer container)
-        {
-            this.workflow = workflow;
-            this.container = container;
-        }
-
-        public string ExtractAccount()
-        {
-            var account = Retry(FetchAccount);
-            if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure.");
-
-            return account;
-        }
-
-        public string ExtractGenesisJsonBase64()
-        {
-            var genesisJson = Retry(FetchGenesisJson);
-            if (string.IsNullOrEmpty(genesisJson)) throw new InvalidOperationException("Unable to fetch genesis-json for geth node. Test infra failure.");
-
-            return Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson));
-        }
-
-        public string ExtractPubKey()
-        {
-            var pubKey = Retry(FetchPubKey);
-            if (string.IsNullOrEmpty(pubKey)) throw new InvalidOperationException("Unable to fetch enode from geth node. Test infra failure.");
-
-            return pubKey;
-        }
-
-        private string Retry(Func<string> fetch)
-        {
-            var result = Catch(fetch);
-            if (string.IsNullOrEmpty(result))
-            {
-                Thread.Sleep(TimeSpan.FromSeconds(5));
-                result = fetch();
-            }
-            return result;
-        }
-
-        private string Catch(Func<string> fetch)
-        {
-            try
-            {
-                return fetch();
-            }
-            catch
-            {
-                return string.Empty;
-            }
-        }
-
-        private string FetchGenesisJson()
-        {
-            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GenesisFilename);
-        }
-
-        private string FetchAccount()
-        {
-            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename);
-        }
-
-        private string FetchPubKey()
-        {
-            var enodeFinder = new PubKeyFinder();
-            workflow.DownloadContainerLog(container, enodeFinder);
-            return enodeFinder.GetPubKey();
-        }
-    }
-
-    public class PubKeyFinder : LogHandler, ILogHandler
-    {
-        private const string openTag = "self=\"enode://";
-        private string pubKey = string.Empty;
-
-        public string GetPubKey()
-        {
-            return pubKey;
-        }
-
-        protected override void ProcessLine(string line)
-        {
-            if (line.Contains(openTag))
-            {
-                ExtractPubKey(line);
-            }
-        }
-
-        private void ExtractPubKey(string line)
-        {
-            var openIndex = line.IndexOf(openTag) + openTag.Length;
-            var closeIndex = line.IndexOf("@");
-
-            pubKey = line.Substring(
-                    startIndex: openIndex,
-                    length: closeIndex - openIndex);
-        }
-    }
-}

DistTestCore/Marketplace/GethStartResult.cs

@@ -1,16 +0,0 @@
-namespace DistTestCore.Marketplace
-{
-    public class GethStartResult
-    {
-        public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo[] companionNodes)
-        {
-            MarketplaceAccessFactory = marketplaceAccessFactory;
-            BootstrapNode = bootstrapNode;
-            CompanionNodes = companionNodes;
-        }
-
-        public IMarketplaceAccessFactory MarketplaceAccessFactory { get; }
-        public GethBootstrapNodeInfo BootstrapNode { get; }
-        public GethCompanionNodeInfo[] CompanionNodes { get; }
-    }
-}

DistTestCore/Marketplace/GethStartupConfig.cs

@@ -1,16 +0,0 @@
-namespace DistTestCore.Marketplace
-{
-    public class GethStartupConfig
-    {
-        public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64, GethBootstrapNodeInfo bootstrapNode)
-        {
-            IsBootstrapNode = isBootstrapNode;
-            GenesisJsonBase64 = genesisJsonBase64;
-            BootstrapNode = bootstrapNode;
-        }
-
-        public bool IsBootstrapNode { get; }
-        public string GenesisJsonBase64 { get; }
-        public GethBootstrapNodeInfo BootstrapNode { get; }
-    }
-}

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -1,79 +0,0 @@
-using Logging;
-using NUnit.Framework;
-using NUnit.Framework.Constraints;
-
-namespace DistTestCore.Marketplace
-{
-    public interface IMarketplaceAccess
-    {
-        void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral);
-        void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes);
-        void AssertThatBalance(IResolveConstraint constraint, string message = "");
-        decimal GetBalance();
-    }
-
-    public class MarketplaceAccess : IMarketplaceAccess
-    {
-        private readonly TestLog log;
-        private readonly GethBootstrapNodeInfo bootstrapNode;
-        private readonly GethCompanionNodeInfo companionNode;
-
-        public MarketplaceAccess(TestLog log, GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo companionNode)
-        {
-            this.log = log;
-            this.bootstrapNode = bootstrapNode;
-            this.companionNode = companionNode;
-        }
-
-        public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes)
-        {
-            throw new NotImplementedException();
-        }
-
-        public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral)
-        {
-            throw new NotImplementedException();
-        }
-
-        public void AssertThatBalance(IResolveConstraint constraint, string message = "")
-        {
-            Assert.That(GetBalance(), constraint, message);
-        }
-
-        public decimal GetBalance()
-        {
-            var interaction = bootstrapNode.StartInteraction(log);
-            return interaction.GetBalance(companionNode.Account);
-        }
-    }
-
-    public class MarketplaceUnavailable : IMarketplaceAccess
-    {
-        public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes)
-        {
-            Unavailable();
-        }
-
-        public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral)
-        {
-            Unavailable();
-        }
-
-        public void AssertThatBalance(IResolveConstraint constraint, string message = "")
-        {
-            Unavailable();
-        }
-
-        public decimal GetBalance()
-        {
-            Unavailable();
-            return 0;
-        }
-
-        private void Unavailable()
-        {
-            Assert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it.");
-            throw new InvalidOperationException();
-        }
-    }
-}

DistTestCore/Marketplace/MarketplaceAccessFactory.cs

@@ -1,42 +0,0 @@
-using DistTestCore.Codex;
-using Logging;
-
-namespace DistTestCore.Marketplace
-{
-    public interface IMarketplaceAccessFactory
-    {
-        IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access);
-    }
-
-    public class MarketplaceUnavailableAccessFactory : IMarketplaceAccessFactory
-    {
-        public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
-        {
-            return new MarketplaceUnavailable();
-        }
-    }
-
-    public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory
-    {
-        private readonly TestLog log;
-        private readonly GethBootstrapNodeInfo bootstrapNode;
-
-        public GethMarketplaceAccessFactory(TestLog log, GethBootstrapNodeInfo bootstrapNode)
-        {
-            this.log = log;
-            this.bootstrapNode = bootstrapNode;
-        }
-
-        public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
-        {
-            var companionNode = GetGethCompanionNode(access);
-            return new MarketplaceAccess(log, bootstrapNode, companionNode);
-        }
-
-        private GethCompanionNodeInfo GetGethCompanionNode(CodexAccess access)
-        {
-            var node = access.Container.Recipe.Additionals.Single(a => a is GethCompanionNodeInfo);
-            return (GethCompanionNodeInfo)node;
-        }
-    }
-}

DistTestCore/Marketplace/MarketplaceInitialConfig.cs

@@ -1,12 +0,0 @@
-namespace DistTestCore.Marketplace
-{
-    public class MarketplaceInitialConfig
-    {
-        public MarketplaceInitialConfig(int initialBalance)
-        {
-            InitialBalance = initialBalance;
-        }
-
-        public int InitialBalance { get; }
-    }
-}

DistTestCore/Metrics/MetricsAccess.cs

@@ -1,70 +0,0 @@
-using KubernetesWorkflow;
-using NUnit.Framework;
-using NUnit.Framework.Constraints;
-using Utils;
-
-namespace DistTestCore.Metrics
-{
-    public interface IMetricsAccess
-    {
-        void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
-    }
-
-    public class MetricsUnavailable : IMetricsAccess
-    {
-        public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
-        {
-            Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
-            throw new InvalidOperationException();
-        }
-    }
-
-    public class MetricsAccess : IMetricsAccess
-    {
-        private readonly MetricsQuery query;
-        private readonly RunningContainer node;
-
-        public MetricsAccess(MetricsQuery query, RunningContainer node)
-        {
-            this.query = query;
-            this.node = node;
-        }
-
-        public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
-        {
-            var metricSet = GetMetricWithTimeout(metricName);
-            var metricValue = metricSet.Values[0].Value;
-            Assert.That(metricValue, constraint, message);
-        }
-
-        public Metrics? GetAllMetrics()
-        {
-            return query.GetAllMetricsForNode(node);
-        }
-
-        private MetricsSet GetMetricWithTimeout(string metricName)
-        {
-            var start = DateTime.UtcNow;
-
-            while (true)
-            {
-                var mostRecent = GetMostRecent(metricName);
-                if (mostRecent != null) return mostRecent;
-                if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
-                {
-                    Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
-                    throw new TimeoutException();
-                }
-
-                Time.Sleep(TimeSpan.FromSeconds(2));
-            }
-        }
-
-        private MetricsSet? GetMostRecent(string metricName)
-        {
-            var result = query.GetMostRecent(metricName, node);
-            if (result == null) return null;
-            return result.Sets.LastOrDefault();
-        }
-    }
-}

DistTestCore/Metrics/MetricsAccessFactory.cs

@@ -1,33 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore.Metrics
-{
-    public interface IMetricsAccessFactory
-    {
-        IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer);
-    }
-
-    public class MetricsUnavailableAccessFactory : IMetricsAccessFactory
-    {
-        public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
-        {
-            return new MetricsUnavailable();
-        }
-    }
-
-    public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory
-    {
-        private readonly RunningContainers prometheusContainer;
-
-        public CodexNodeMetricsAccessFactory(RunningContainers prometheusContainer)
-        {
-            this.prometheusContainer = prometheusContainer;
-        }
-
-        public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
-        {
-            var query = new MetricsQuery(prometheusContainer);
-            return new MetricsAccess(query, codexContainer);
-        }
-    }
-}

DistTestCore/Metrics/PrometheusContainerRecipe.cs

@@ -1,19 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore.Metrics
-{
-    public class PrometheusContainerRecipe : ContainerRecipeFactory
-    {
-        public const string DockerImage = "thatbenbierens/prometheus-envconf:latest";
-
-        protected override string Image => DockerImage;
-
-        protected override void Initialize(StartupConfig startupConfig)
-        {
-            var config = startupConfig.Get<PrometheusStartupConfig>();
-
-            AddExposedPortAndVar("PROM_PORT");
-            AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
-        }
-    }
-}

DistTestCore/OnlineCodexNode.cs

@@ -1,133 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Logs;
-using DistTestCore.Marketplace;
-using DistTestCore.Metrics;
-using NUnit.Framework;
-
-namespace DistTestCore
-{
-    public interface IOnlineCodexNode
-    {
-        CodexDebugResponse GetDebugInfo();
-        ContentId UploadFile(TestFile file);
-        TestFile? DownloadContent(ContentId contentId);
-        void ConnectToPeer(IOnlineCodexNode node);
-        ICodexNodeLog DownloadLog();
-        IMetricsAccess Metrics { get; }
-        IMarketplaceAccess Marketplace { get; }
-    }
-
-    public class OnlineCodexNode : IOnlineCodexNode
-    {
-        private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
-        private const string UploadFailedMessage = "Unable to store block";
-        private readonly TestLifecycle lifecycle;
-
-        public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess)
-        {
-            this.lifecycle = lifecycle;
-            CodexAccess = codexAccess;
-            Group = group;
-            Metrics = metricsAccess;
-            Marketplace = marketplaceAccess;
-        }
-
-        public CodexAccess CodexAccess { get; }
-        public CodexNodeGroup Group { get; }
-        public IMetricsAccess Metrics { get; }
-        public IMarketplaceAccess Marketplace { get; }
-
-        public string GetName()
-        {
-            return $"<{CodexAccess.Container.Recipe.Name}>";
-        }
-
-        public CodexDebugResponse GetDebugInfo()
-        {
-            var response = CodexAccess.GetDebugInfo();
-            Log($"Got DebugInfo with id: '{response.id}'.");
-            return response;
-        }
-
-        public ContentId UploadFile(TestFile file)
-        {
-            Log($"Uploading file of size {file.GetFileSize()}...");
-            using var fileStream = File.OpenRead(file.Filename);
-            var response = CodexAccess.UploadFile(fileStream);
-            if (response.StartsWith(UploadFailedMessage))
-            {
-                Assert.Fail("Node failed to store block.");
-            }
-            Log($"Uploaded file. Received contentId: '{response}'.");
-            return new ContentId(response);
-        }
-
-        public TestFile? DownloadContent(ContentId contentId)
-        {
-            Log($"Downloading for contentId: '{contentId.Id}'...");
-            var file = lifecycle.FileManager.CreateEmptyTestFile();
-            DownloadToFile(contentId.Id, file);
-            Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'.");
-            return file;
-        }
-
-        public void ConnectToPeer(IOnlineCodexNode node)
-        {
-            var peer = (OnlineCodexNode)node;
-
-            Log($"Connecting to peer {peer.GetName()}...");
-            var peerInfo = node.GetDebugInfo();
-            var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
-
-            Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes.");
-            Log($"Successfully connected to peer {peer.GetName()}.");
-        }
-
-        public ICodexNodeLog DownloadLog()
-        {
-            return lifecycle.DownloadLog(this);
-        }
-
-        public string Describe()
-        {
-            return $"({Group.Describe()} contains {GetName()})";
-        }
-
-        private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
-        {
-            var multiAddress = peerInfo.addrs.First();
-            // Todo: Is there a case where First address in list is not the way?
-
-            if (Group == peer.Group)
-            {
-                return multiAddress;
-            }
-
-            // The peer we want to connect is in a different pod.
-            // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.Ip);
-        }
-
-        private void DownloadToFile(string contentId, TestFile file)
-        {
-            using var fileStream = File.OpenWrite(file.Filename);
-            using var downloadStream = CodexAccess.DownloadFile(contentId);
-            downloadStream.CopyTo(fileStream);
-        }
-
-        private void Log(string msg)
-        {
-            lifecycle.Log.Log($"{GetName()}: {msg}");
-        }
-    }
-
-    public class ContentId
-    {
-        public ContentId(string id)
-        {
-            Id = id;
-        }
-
-        public string Id { get; }
-    }
-}

DistTestCore/PrometheusStarter.cs

@@ -1,65 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Metrics;
-using KubernetesWorkflow;
-using System.Text;
-
-namespace DistTestCore
-{
-    public class PrometheusStarter
-    {
-        private readonly TestLifecycle lifecycle;
-        private readonly WorkflowCreator workflowCreator;
-
-        public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-        {
-            this.lifecycle = lifecycle;
-            this.workflowCreator = workflowCreator;
-        }
-
-        public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers)
-        {
-            if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
-
-            Log($"Starting metrics server for {containers.Describe()}");
-            var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers)));
-
-            var workflow = workflowCreator.CreateWorkflow();
-            var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
-            if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
-
-            Log("Metrics server started.");
-
-            return new CodexNodeMetricsAccessFactory(runningContainers);
-        }
-
-        private string GeneratePrometheusConfig(RunningContainer[] nodes)
-        {
-            var config = "";
-            config += "global:\n";
-            config += "  scrape_interval: 30s\n";
-            config += "  scrape_timeout: 10s\n";
-            config += "\n";
-            config += "scrape_configs:\n";
-            config += "  - job_name: services\n";
-            config += "    metrics_path: /metrics\n";
-            config += "    static_configs:\n";
-            config += "      - targets:\n";
-
-            foreach (var node in nodes)
-            {
-                var ip = node.Pod.Ip;
-                var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
-                config += $"          - '{ip}:{port}'\n";
-            }
-
-            var bytes = Encoding.ASCII.GetBytes(config);
-            return Convert.ToBase64String(bytes);
-        }
-
-        private void Log(string msg)
-        {
-            lifecycle.Log.Log(msg);
-        }
-    }
-}

DistTestCore/Stopwatch.cs

@@ -1,19 +0,0 @@
-using Logging;
-using Utils;
-
-namespace DistTestCore
-{
-    public class Stopwatch
-    {
-        public static void Measure(BaseLog log, string name, Action action)
-        {
-            var start = DateTime.UtcNow;
-
-            action();
-
-            var duration = DateTime.UtcNow - start;
-
-            log.Log($"{name} ({Time.FormatDuration(duration)})");
-        }
-    }
-}

DistTestCore/TestLifecycle.cs

@@ -1,46 +0,0 @@
-using DistTestCore.Logs;
-using KubernetesWorkflow;
-using Logging;
-
-namespace DistTestCore
-{
-    public class TestLifecycle
-    {
-        private readonly WorkflowCreator workflowCreator;
-
-        public TestLifecycle(TestLog log, Configuration configuration)
-        {
-            Log = log;
-            workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration());
-
-            FileManager = new FileManager(Log, configuration);
-            CodexStarter = new CodexStarter(this, workflowCreator);
-            PrometheusStarter = new PrometheusStarter(this, workflowCreator);
-            GethStarter = new GethStarter(this, workflowCreator);
-        }
-
-        public TestLog Log { get; }
-        public FileManager FileManager { get; }
-        public CodexStarter CodexStarter { get; }
-        public PrometheusStarter PrometheusStarter { get; }
-        public GethStarter GethStarter { get; }
-
-        public void DeleteAllResources()
-        {
-            CodexStarter.DeleteAllResources();
-            FileManager.DeleteAllTestFiles();
-        }
-
-        public ICodexNodeLog DownloadLog(OnlineCodexNode node)
-        {
-            var subFile = Log.CreateSubfile();
-            var description = node.Describe();
-            var handler = new LogDownloadHandler(description, subFile);
-
-            Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-            CodexStarter.DownloadLog(node.CodexAccess.Container, handler);
-
-            return new CodexNodeLog(subFile);
-        }
-    }
-}

DistTestCore/Timing.cs

@@ -1,132 +0,0 @@
-using NUnit.Framework;
-using Utils;
-
-namespace DistTestCore
-{
-    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
-    public class UseLongTimeoutsAttribute : PropertyAttribute
-    {
-        public UseLongTimeoutsAttribute()
-            : base(Timing.UseLongTimeoutsKey)
-        {
-        }
-    }
-
-    public static class Timing
-    {
-        public const string UseLongTimeoutsKey = "UseLongTimeouts";
-
-        public static TimeSpan HttpCallTimeout()
-        {
-            return GetTimes().HttpCallTimeout();
-        }
-
-        public static int HttpCallRetryCount()
-        {
-            return GetTimes().HttpCallRetryCount();
-        }
-
-        public static void HttpCallRetryDelay()
-        {
-            Time.Sleep(GetTimes().HttpCallRetryDelay());
-        }
-
-        public static TimeSpan K8sServiceDelay()
-        {
-            return GetTimes().WaitForK8sServiceDelay();
-        }
-
-        public static TimeSpan K8sOperationTimeout()
-        {
-            return GetTimes().K8sOperationTimeout();
-        }
-
-        public static TimeSpan WaitForMetricTimeout()
-        {
-            return GetTimes().WaitForMetricTimeout();
-        }
-
-        private static ITimeSet GetTimes()
-        {
-            var testProperties = TestContext.CurrentContext.Test.Properties;
-            if (testProperties.ContainsKey(UseLongTimeoutsKey)) return new LongTimeSet();
-            return new DefaultTimeSet();
-        }
-    }
-
-    public interface ITimeSet
-    {
-        TimeSpan HttpCallTimeout();
-        int HttpCallRetryCount();
-        TimeSpan HttpCallRetryDelay();
-        TimeSpan WaitForK8sServiceDelay();
-        TimeSpan K8sOperationTimeout();
-        TimeSpan WaitForMetricTimeout();
-    }
-
-    public class DefaultTimeSet : ITimeSet
-    {
-        public TimeSpan HttpCallTimeout()
-        {
-            return TimeSpan.FromSeconds(10);
-        }
-
-        public int HttpCallRetryCount()
-        {
-            return 5;
-        }
-
-        public TimeSpan HttpCallRetryDelay()
-        {
-            return TimeSpan.FromSeconds(3);
-        }
-
-        public TimeSpan WaitForK8sServiceDelay()
-        {
-            return TimeSpan.FromSeconds(1);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromMinutes(5);
-        }
-
-        public TimeSpan WaitForMetricTimeout()
-        {
-            return TimeSpan.FromSeconds(30);
-        }
-    }
-
-    public class LongTimeSet : ITimeSet
-    {
-        public TimeSpan HttpCallTimeout()
-        {
-            return TimeSpan.FromHours(2);
-        }
-
-        public int HttpCallRetryCount()
-        {
-            return 2;
-        }
-
-        public TimeSpan HttpCallRetryDelay()
-        {
-            return TimeSpan.FromMinutes(5);
-        }
-
-        public TimeSpan WaitForK8sServiceDelay()
-        {
-            return TimeSpan.FromSeconds(10);
-        }
-
-        public TimeSpan K8sOperationTimeout()
-        {
-            return TimeSpan.FromMinutes(15);
-        }
-
-        public TimeSpan WaitForMetricTimeout()
-        {
-            return TimeSpan.FromMinutes(5);
-        }
-    }
-}

Framework/ArgsUniform/ArgsUniform.cs

@@ -0,0 +1,130 @@
+using System.Reflection;
+
+namespace ArgsUniform
+{
+    public class ArgsUniform<T>
+    {
+        private readonly Assigner<T> assigner;
+        private readonly Action printAppInfo;
+        private readonly string[] args;
+        private const int cliStart = 8;
+        private const int shortStart = 38;
+        private const int envStart = 48;
+        private const int descStart = 80;
+
+        public ArgsUniform(Action printAppInfo, params string[] args)
+            : this(printAppInfo, new IEnv.Env(), args)
+        {
+        }
+
+        public ArgsUniform(Action printAppInfo, object defaultsProvider, params string[] args)
+            : this(printAppInfo, defaultsProvider, new IEnv.Env(), args)
+        {
+        }
+
+        public ArgsUniform(Action printAppInfo, IEnv.IEnv env, params string[] args)
+            : this(printAppInfo, null!, env, args)
+        {
+        }
+
+        public ArgsUniform(Action printAppInfo, object defaultsProvider, IEnv.IEnv env, params string[] args)
+        {
+            this.printAppInfo = printAppInfo;
+            this.args = args;
+
+            assigner = new Assigner<T>(env, args, defaultsProvider);
+        }
+
+        public T Parse(bool printResult = false)
+        {
+            if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
+            {
+                printAppInfo();
+                PrintHelp();
+                Environment.Exit(0);
+            }
+
+            var result = Activator.CreateInstance<T>();
+            var uniformProperties = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
+            var missingRequired = new List<PropertyInfo>();
+            foreach (var uniformProperty in uniformProperties)
+            {
+                var attr = uniformProperty.GetCustomAttribute<UniformAttribute>();
+                if (attr != null)
+                {
+                    if (!assigner.UniformAssign(result, attr, uniformProperty) && attr.Required)
+                    {
+                        missingRequired.Add(uniformProperty);
+                    }
+                }
+            }
+
+            if (missingRequired.Any())
+            {
+                PrintResults(printResult,result, uniformProperties);
+                Print("");
+                foreach (var missing in missingRequired)
+                {
+                    var attr = missing.GetCustomAttribute<UniformAttribute>()!;
+                    var exampleArg = $"--{attr.Arg}=...";
+                    var exampleEnvVar = $"{attr.EnvVar}=...";
+                    Print($" ! Missing required input. Use argument: '{exampleArg}' or environment variable: '{exampleEnvVar}'.");
+                }
+
+                PrintHelp();
+                Environment.Exit(1);
+            }
+
+            PrintResults(printResult, result, uniformProperties);
+
+            return result;
+        }
+
+        public void PrintHelp()
+        {
+            Print("");
+            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description", "(default)");
+            var props = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
+            foreach (var prop in props)
+            {
+                var a = prop.GetCustomAttribute<UniformAttribute>();
+                if (a != null)
+                {
+                    var optional = !a.Required ? " (optional)" : "";
+                    var def = assigner.DescribeDefaultFor(prop);
+                    PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional, $"({def})");
+                }
+            }
+            Print("");
+        }
+
+        private void PrintResults(bool printResult, T result, PropertyInfo[] uniformProperties)
+        {
+            if (!printResult) return;
+            Print("");
+            foreach (var p in uniformProperties)
+            {
+                Print($"\t{p.Name} = {p.GetValue(result)}");
+            }
+            Print("");
+        }
+
+        private void Print(string msg)
+        {
+            Console.WriteLine(msg);
+        }
+
+        private void PrintAligned(string cli, string s, string env, string desc, string def)
+        {
+            Console.CursorLeft = cliStart;
+            Console.Write(cli);
+            Console.CursorLeft = shortStart;
+            Console.Write(s);
+            Console.CursorLeft = envStart;
+            Console.Write(env);
+            Console.CursorLeft = descStart;
+            Console.Write(desc + " ");
+            Console.Write(def + Environment.NewLine);
+        }
+    }
+}

Framework/ArgsUniform/ArgsUniform.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="I-Env" Version="1.0.0" />
+  </ItemGroup>
+
+</Project>

Framework/ArgsUniform/Assigner.cs

@@ -0,0 +1,186 @@
+using System.Globalization;
+using System.Numerics;
+using System.Reflection;
+
+namespace ArgsUniform
+{
+    public class Assigner<T>
+    {
+        private readonly IEnv.IEnv env;
+        private readonly string[] args;
+        private readonly object? defaultsProvider;
+
+        public Assigner(IEnv.IEnv env, string[] args, object? defaultsProvider)
+        {
+            this.env = env;
+            this.args = args;
+            this.defaultsProvider = defaultsProvider;
+        }
+
+        public bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
+            return false;
+        }
+
+        public string DescribeDefaultFor(PropertyInfo property)
+        {
+            var obj = Activator.CreateInstance<T>();
+            var defaultValue = GetDefaultValue(obj, property);
+            if (defaultValue == null) return "";
+            if (defaultValue is string str)
+            {
+                return "\"" + str + "\"";
+            }
+            return defaultValue.ToString() ?? string.Empty;
+        }
+
+        private object? GetDefaultValue(T result, PropertyInfo uniformProperty)
+        {
+            // Get value from object's static initializer if it's there.
+            var currentValue = uniformProperty.GetValue(result);
+            if (currentValue != null) return currentValue;
+
+            // Get value from defaults-provider object if it's there.
+            if (defaultsProvider == null) return null;
+            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
+            if (defaultProperty == null) return null;
+            return defaultProperty.GetValue(defaultsProvider);
+        }
+
+        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
+        {
+            var defaultValue = GetDefaultValue(result, uniformProperty);
+            var isEmptryString = (defaultValue as string) == string.Empty;
+            if (defaultValue != null && defaultValue != GetDefaultValueForType(uniformProperty.PropertyType) && !isEmptryString)
+            {
+                return Assign(result, uniformProperty, defaultValue);
+            }
+            return false;
+        }
+
+        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
+            if (!string.IsNullOrEmpty(e))
+            {
+                return Assign(result, uniformProperty, e);
+            }
+            return false;
+        }
+
+        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var fromArg = GetFromArgs(attr.Arg);
+            if (fromArg != null)
+            {
+                return Assign(result, uniformProperty, fromArg);
+            }
+            var fromShort = GetFromArgs(attr.ArgShort);
+            if (fromShort != null)
+            {
+                return Assign(result, uniformProperty, fromShort);
+            }
+            return false;
+        }
+
+        private bool Assign(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (uniformProperty.PropertyType == value.GetType())
+            {
+                uniformProperty.SetValue(result, value);
+                return true;
+            }
+            else
+            {
+                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
+                {
+                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
+                    return true;
+                }
+                else
+                {
+                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(ulong)) return AssignUlong(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(BigInteger)) return AssignBigInt(result, uniformProperty, value);
+
+                    throw new NotSupportedException(
+                        $"Unsupported property type '${uniformProperty.PropertyType}' " +
+                        $"for property '${uniformProperty.Name}'.");
+                }
+            }
+        }
+
+        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
+            {
+                uniformProperty.SetValue(result, e);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (int.TryParse(value.ToString(), CultureInfo.InvariantCulture, out int i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignUlong(T? result, PropertyInfo uniformProperty, object value)
+        {
+            if (ulong.TryParse(value.ToString(), CultureInfo.InvariantCulture, out ulong i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private bool AssignBigInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (BigInteger.TryParse(value.ToString(), CultureInfo.InvariantCulture, out BigInteger i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
+            {
+                uniformProperty.SetValue(result, true);
+            }
+            return true;
+        }
+
+        private string? GetFromArgs(string key)
+        {
+            var argKey = $"--{key}=";
+            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
+            if (arg != null)
+            {
+                return arg.Substring(argKey.Length);
+            }
+            return null;
+        }
+
+        private static object GetDefaultValueForType(Type t)
+        {
+            if (t.IsValueType) return Activator.CreateInstance(t)!;
+            return null!;
+        }
+    }
+}

Framework/ArgsUniform/ExampleUser.cs

@@ -0,0 +1,34 @@
+namespace ArgsUniform
+{
+    public class ExampleUser
+    {
+        public class Args
+        {
+            [Uniform("aaa", "a", "AAA", false, "Sets the AAA!")]
+            public string Aaa { get; set; } = string.Empty;
+
+            [Uniform("bbb", "b", "BBB", true, "Sets that BBB")]
+            public string Bbb { get; set; } = string.Empty;
+        }
+
+        public class DefaultsProvider
+        {
+            public string Aaa { get { return "non-static operation"; } }
+        }
+
+        public void Example()
+        {
+            // env var: "AAA=BBB"
+            var args = "--ccc=ddd";
+
+            var uniform = new ArgsUniform<Args>(PrintHelp, new DefaultsProvider(), args);
+
+            var aaa = uniform.Parse();
+        }
+
+        private static void PrintHelp()
+        {
+            Console.WriteLine("Help text!");
+        }
+    }
+}

Framework/ArgsUniform/UniformAttribute.cs

@@ -0,0 +1,20 @@
+namespace ArgsUniform
+{
+    public class UniformAttribute : Attribute
+    {
+        public UniformAttribute(string arg, string argShort, string envVar, bool required, string description)
+        {
+            Arg = arg;
+            ArgShort = argShort;
+            EnvVar = envVar;
+            Required = required;
+            Description = description;
+        }
+
+        public string Arg { get; }
+        public string ArgShort { get; }
+        public string EnvVar { get; }
+        public bool Required { get; }
+        public string Description { get; }
+    }
+}

Framework/Core/Core.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\FileUtils\FileUtils.csproj" />
+    <ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/Core/CoreInterface.cs

@@ -0,0 +1,56 @@
+using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
+
+namespace Core
+{
+    public sealed class CoreInterface
+    {
+        private readonly EntryPoint entryPoint;
+
+        internal CoreInterface(EntryPoint entryPoint)
+        {
+            this.entryPoint = entryPoint;
+        }
+
+        public T GetPlugin<T>() where T : IProjectPlugin
+        {
+            return entryPoint.GetPlugin<T>();
+        }
+
+        public IKnownLocations GetKnownLocations()
+        {
+            return entryPoint.Tools.CreateWorkflow().GetAvailableLocations();
+        }
+
+        public IDownloadedLog DownloadLog(IHasContainer containerSource, int? tailLines = null)
+        {
+            return DownloadLog(containerSource.Container, tailLines);
+        }
+
+        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
+        {
+            var workflow = entryPoint.Tools.CreateWorkflow();
+            var msg = $"Downloading container log for '{container.Name}'";
+            entryPoint.Tools.GetLog().Log(msg);
+            var logHandler = new WriteToFileLogHandler(entryPoint.Tools.GetLog(), msg);
+            workflow.DownloadContainerLog(container, logHandler, tailLines);
+            return new DownloadedLog(logHandler);
+        }
+
+        public string ExecuteContainerCommand(IHasContainer containerSource, string command, params string[] args)
+        {
+            return ExecuteContainerCommand(containerSource.Container, command, args);
+        }
+
+        public string ExecuteContainerCommand(RunningContainer container, string command, params string[] args)
+        {
+            var workflow = entryPoint.Tools.CreateWorkflow();
+            return workflow.ExecuteCommand(container, command, args);
+        }
+    }
+
+    public interface IHasContainer
+    {
+        RunningContainer Container { get; }
+    }
+}

Framework/Core/DownloadedLog.cs

@@ -0,0 +1,80 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace Core
+{
+    public interface IDownloadedLog
+    {
+        void IterateLines(Action<string> action);
+        string[] GetLinesContaining(string expectedString);
+        string[] FindLinesThatContain(params string[] tags);
+        void DeleteFile();
+    }
+
+    internal class DownloadedLog : IDownloadedLog
+    {
+        private readonly LogFile logFile;
+
+        internal DownloadedLog(WriteToFileLogHandler logHandler)
+        {
+            logFile = logHandler.LogFile;
+        }
+
+        public void IterateLines(Action<string> action)
+        {
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                action(line);
+                line = streamReader.ReadLine();
+            }
+        }
+
+        public string[] GetLinesContaining(string expectedString)
+        {
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+            var lines = new List<string>();
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                if (line.Contains(expectedString))
+                {
+                    lines.Add(line);
+                }
+                line = streamReader.ReadLine();
+            }
+
+            return lines.ToArray(); ;
+        }
+
+        public string[] FindLinesThatContain(params string[] tags)
+        {
+            var result = new List<string>();
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                if (tags.All(line.Contains))
+                {
+                    result.Add(line);
+                }
+
+                line = streamReader.ReadLine();
+            }
+
+            return result.ToArray();
+        }
+
+        public void DeleteFile()
+        {
+            File.Delete(logFile.FullFilename);
+        }
+    }
+}

Framework/Core/Endpoint.cs

@@ -0,0 +1,193 @@
+using Logging;
+using Newtonsoft.Json;
+using Serialization = Newtonsoft.Json.Serialization;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using Utils;
+
+namespace Core
+{
+    public interface IEndpoint
+    {
+        string HttpGetString(string route);
+        T HttpGetJson<T>(string route);
+        TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
+        string HttpPostJson<TRequest>(string route, TRequest body);
+        TResponse HttpPostString<TResponse>(string route, string body);
+        string HttpPostStream(string route, Stream stream);
+        Stream HttpGetStream(string route);
+        T Deserialize<T>(string json);
+    }
+
+    internal class Endpoint : IEndpoint
+    {
+        private readonly ILog log;
+        private readonly IHttp http;
+        private readonly Address address;
+        private readonly string baseUrl;
+        private readonly string? logAlias;
+
+        public Endpoint(ILog log, IHttp http, Address address, string baseUrl, string? logAlias)
+        {
+            this.log = log;
+            this.http = http;
+            this.address = address;
+            this.baseUrl = baseUrl;
+            this.logAlias = logAlias;
+        }
+
+        public string HttpGetString(string route)
+        {
+            return http.OnClient(client =>
+            {
+                return GetString(client, route);
+            }, $"HTTP-GET:{route}");
+        }
+
+        public T HttpGetJson<T>(string route)
+        {
+            return http.OnClient(client =>
+            {
+                var json = GetString(client, route);
+                return Deserialize<T>(json);
+            }, $"HTTP-GET:{route}");
+        }
+
+        public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJson(client, route, body);
+                var json = Time.Wait(response.Content.ReadAsStringAsync());
+                if (!response.IsSuccessStatusCode)
+                {
+                    throw new HttpRequestException(json);
+                }
+                Log(GetUrl() + route, json);
+                return Deserialize<TResponse>(json);
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        public string HttpPostJson<TRequest>(string route, TRequest body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJson(client, route, body);
+                return Time.Wait(response.Content.ReadAsStringAsync());
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        public TResponse HttpPostString<TResponse>(string route, string body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJsonString(client, route, body);
+                if (response == null) throw new Exception("Received no response.");
+                var result = Deserialize<TResponse>(response);
+                if (result == null) throw new Exception("Failed to deserialize response");
+                return result;
+            }, $"HTTO-POST-JSON: {route}");
+        }
+
+        public string HttpPostStream(string route, Stream stream)
+        {
+            return http.OnClient(client =>
+            {
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                var content = new StreamContent(stream);
+                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
+                var response = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(response.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STREAM: {route}");
+        }
+
+        public Stream HttpGetStream(string route)
+        {
+            return http.OnClient(client =>
+            {
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                return Time.Wait(client.GetStreamAsync(url));
+            }, $"HTTP-GET-STREAM: {route}");
+        }
+
+        public T Deserialize<T>(string json)
+        {
+            var errors = new List<string>();
+            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
+            {
+                Error = delegate (object? sender, Serialization.ErrorEventArgs args)
+                {
+                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
+                    {
+                        errors.Add($"""
+                                    Member: '{args.ErrorContext.Member?.ToString() ?? "<null>"}'
+                                    Path: {args.ErrorContext.Path}
+                                    Error: {args.ErrorContext.Error.Message}
+                                    """);
+                        args.ErrorContext.Handled = true;
+                    }
+                }
+            });
+            if (errors.Count > 0)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
+            }
+            else if (deserialized == null)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
+            }
+            return deserialized;
+        }
+
+        private string GetString(HttpClient client, string route)
+        {
+            var url = GetUrl() + route;
+            Log(url, "");
+            var result = Time.Wait(client.GetAsync(url));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
+        private HttpResponseMessage PostJson<TRequest>(HttpClient client, string route, TRequest body)
+        {
+            var url = GetUrl() + route;
+            using var content = JsonContent.Create(body);
+            Log(url, JsonConvert.SerializeObject(body));
+            return Time.Wait(client.PostAsync(url, content));
+        }
+
+        private string PostJsonString(HttpClient client, string route, string body)
+        {
+            var url = GetUrl() + route;
+            Log(url, body);
+            var content = new StringContent(body);
+            content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+            var result = Time.Wait(client.PostAsync(url, content));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
+        private string GetUrl()
+        {
+            return $"{address.Host}:{address.Port}{baseUrl}";
+        }
+
+        private void Log(string url, string message)
+        {
+            if (logAlias != null)
+            {
+                log.Debug($"({logAlias})({url}) = '{message}'", 3);
+            }
+            else
+            {
+                log.Debug($"({url}) = '{message}'", 3);
+            }
+        }
+    }
+}

Framework/Core/EntryPoint.cs

@@ -0,0 +1,56 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace Core
+{
+    public class EntryPoint
+    {
+        private readonly IToolsFactory toolsFactory;
+        private readonly PluginManager manager = new PluginManager();
+
+        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        {
+            toolsFactory = new ToolsFactory(log, configuration, fileManagerRootFolder, timeSet);
+
+            Tools = toolsFactory.CreateTools();
+            manager.InstantiatePlugins(PluginFinder.GetPluginTypes(), toolsFactory);
+        }
+
+        public EntryPoint(ILog log, Configuration configuration, string fileManagerRootFolder)
+            : this(log, configuration, fileManagerRootFolder, new DefaultTimeSet())
+        {
+        }
+
+        public IPluginTools Tools { get; }
+
+        public void Announce()
+        {
+            manager.AnnouncePlugins();
+        }
+
+        public Dictionary<string, string> GetPluginMetadata()
+        {
+            return manager.GatherPluginMetadata().Get();
+        }
+
+        public CoreInterface CreateInterface()
+        {
+            return new CoreInterface(this);
+        }
+
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
+        {
+            manager.DecommissionPlugins(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
+            Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
+        }
+
+        internal T GetPlugin<T>() where T : IProjectPlugin
+        {
+            return manager.GetPlugin<T>();
+        }
+    }
+}

Framework/Core/Http.cs

@@ -0,0 +1,84 @@
+using Logging;
+using Utils;
+
+namespace Core
+{
+    public interface IHttp
+    {
+        T OnClient<T>(Func<HttpClient, T> action);
+        T OnClient<T>(Func<HttpClient, T> action, string description);
+        T OnClient<T>(Func<HttpClient, T> action, Retry retry);
+        IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null);
+    }
+
+    internal class Http : IHttp
+    {
+        private static readonly object httpLock = new object();
+        private readonly ILog log;
+        private readonly ITimeSet timeSet;
+        private readonly Action<HttpClient> onClientCreated;
+
+        internal Http(ILog log, ITimeSet timeSet)
+            : this(log, timeSet, DoNothing)
+        {
+        }
+
+        internal Http(ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
+        {
+            this.log = log;
+            this.timeSet = timeSet;
+            this.onClientCreated = onClientCreated;
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action)
+        {
+            return OnClient(action, GetDescription());
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action, string description)
+        {
+            var retry = new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), f => { });
+            return OnClient(action, retry);
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action, Retry retry)
+        {
+            var client = GetClient();
+
+            return LockRetry(() =>
+            {
+                return action(client);
+            }, retry);
+        }
+
+        public IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null)
+        {
+            return new Endpoint(log, this, address, baseUrl, logAlias);
+        }
+
+        private string GetDescription()
+        {
+            return DebugStack.GetCallerName(skipFrames: 2);
+        }
+
+        private T LockRetry<T>(Func<T> operation, Retry retry)
+        {
+            lock (httpLock)
+            {
+                return retry.Run(operation);
+            }
+        }
+
+        private HttpClient GetClient()
+        {
+            var client = new HttpClient();
+            client.Timeout = timeSet.HttpCallTimeout();
+            onClientCreated(client);
+            return client;
+        }
+
+        private static void DoNothing(HttpClient client)
+        {
+        }
+    }
+}

Framework/Core/PluginFinder.cs

@@ -0,0 +1,46 @@
+using System.Reflection;
+
+namespace Core
+{
+    internal static class PluginFinder
+    {
+        private static Type[]? pluginTypes = null;
+
+        internal static Type[] GetPluginTypes()
+        {
+            if (pluginTypes != null) return pluginTypes;
+
+            // Reflection can be costly. Do this only once.
+            FindAndLoadPluginAssemblies();
+
+            var assemblies = AppDomain.CurrentDomain.GetAssemblies();
+            pluginTypes = assemblies.SelectMany(a => a.GetTypes().Where(t =>
+                typeof(IProjectPlugin).IsAssignableFrom(t) &&
+                !t.IsAbstract)
+            ).ToArray();
+
+            return pluginTypes;
+        }
+
+        private static void FindAndLoadPluginAssemblies()
+        {
+            var files = Directory.GetFiles(".");
+            foreach (var file in files)
+            {
+                var f = file.ToLowerInvariant();
+                if (f.Contains("plugin") && f.EndsWith("dll"))
+                {
+                    var name = Path.GetFileNameWithoutExtension(file);
+                    try
+                    {
+                        Assembly.Load(name);
+                    }
+                    catch (Exception ex)
+                    {
+                        throw new Exception($"Failed to load plugin from file '{name}'.", ex);
+                    }
+                }
+            }
+        }
+    }
+}

Framework/Core/PluginManager.cs

@@ -0,0 +1,78 @@
+namespace Core
+{
+    internal class PluginManager
+    {
+        private readonly List<PluginToolsPair> pairs = new List<PluginToolsPair>();
+
+        internal void InstantiatePlugins(Type[] pluginTypes, IToolsFactory provider)
+        {
+            pairs.Clear();
+            foreach (var pluginType in pluginTypes)
+            {
+                var tools = provider.CreateTools();
+                var plugin = InstantiatePlugins(pluginType, tools);
+
+                ApplyLogPrefix(plugin, tools);
+            }
+        }
+
+        internal void AnnouncePlugins()
+        {
+            foreach (var pair in pairs) pair.Plugin.Announce();
+        }
+
+        internal PluginMetadata GatherPluginMetadata()
+        {
+            var metadata = new PluginMetadata();
+            foreach (var pair in pairs)
+            {
+                if (pair.Plugin is IHasMetadata m)
+                {
+                    m.AddMetadata(metadata);
+                }
+            }
+            return metadata;
+        }
+
+        internal void DecommissionPlugins(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
+        {
+            foreach (var pair in pairs)
+            {
+                pair.Plugin.Decommission();
+                pair.Tools.Decommission(deleteKubernetesResources, deleteTrackedFiles, waitTillDone);
+            }
+        }
+
+        internal T GetPlugin<T>() where T : IProjectPlugin
+        {
+            return (T)pairs.Single(p => p.Plugin.GetType() == typeof(T)).Plugin;
+        }
+
+        private IProjectPlugin InstantiatePlugins(Type pluginType, PluginTools tools)
+        {
+            var plugin = (IProjectPlugin)Activator.CreateInstance(pluginType, args: tools)!;
+            pairs.Add(new PluginToolsPair(plugin, tools));
+            return plugin;
+        }
+
+        private void ApplyLogPrefix(IProjectPlugin plugin, PluginTools tools)
+        {
+            if (plugin is IHasLogPrefix hasLogPrefix)
+            {
+                tools.ApplyLogPrefix(hasLogPrefix.LogPrefix);
+            }
+        }
+
+        private class PluginToolsPair
+        {
+            public PluginToolsPair(IProjectPlugin plugin, IPluginTools tools)
+            {
+                Plugin = plugin;
+                Tools = tools;
+            }
+
+            public IProjectPlugin Plugin { get; }
+            public IPluginTools Tools { get; }
+        }
+    }
+}

Framework/Core/PluginMetadata.cs

@@ -0,0 +1,27 @@
+namespace Core
+{
+    internal interface IPluginMetadata
+    {
+        Dictionary<string, string> Get();
+    }
+
+    public interface IAddMetadata
+    {
+        void Add(string key, string value);
+    }
+
+    internal class PluginMetadata : IPluginMetadata, IAddMetadata
+    {
+        private readonly Dictionary<string, string> metadata = new Dictionary<string, string>();
+
+        public void Add(string key, string value)
+        {
+            metadata.Add(key, value);
+        }
+
+        public Dictionary<string, string> Get()
+        {
+            return new Dictionary<string, string>(metadata);
+        }
+    }
+}

Framework/Core/PluginTools.cs

@@ -0,0 +1,97 @@
+using FileUtils;
+using KubernetesWorkflow;
+using Logging;
+
+namespace Core
+{
+    public interface IPluginTools : IWorkflowTool, ILogTool, IHttpFactoryTool, IFileTool
+    {
+        ITimeSet TimeSet { get; }
+
+        /// <summary>
+        /// Deletes kubernetes and tracked file resources.
+        /// when `waitTillDone` is true, this function will block until resources are deleted.
+        /// </summary>
+        void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone);
+    }
+
+    public interface IWorkflowTool
+    {
+        IStartupWorkflow CreateWorkflow(string? namespaceOverride = null);
+    }
+
+    public interface ILogTool
+    {
+        ILog GetLog();
+    }
+
+    public interface IHttpFactoryTool
+    {
+        IHttp CreateHttp(Action<HttpClient> onClientCreated);
+        IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
+        IHttp CreateHttp();
+    }
+
+    public interface IFileTool
+    {
+        IFileManager GetFileManager();
+    }
+
+    internal class PluginTools : IPluginTools
+    {
+        private readonly WorkflowCreator workflowCreator;
+        private readonly IFileManager fileManager;
+        private readonly LogPrefixer log;
+
+        internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet)
+        {
+            this.log = new LogPrefixer(log);
+            this.workflowCreator = workflowCreator;
+            TimeSet = timeSet;
+            fileManager = new FileManager(log, fileManagerRootFolder);
+        }
+
+        public ITimeSet TimeSet { get; }
+
+        public void ApplyLogPrefix(string prefix)
+        {
+            log.Prefix = prefix;
+        }
+
+        public IHttp CreateHttp(Action<HttpClient> onClientCreated)
+        {
+            return CreateHttp(onClientCreated, TimeSet);
+        }
+
+        public IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet ts)
+        {
+            return new Http(log, ts, onClientCreated);
+        }
+
+        public IHttp CreateHttp()
+        {
+            return new Http(log, TimeSet);
+        }
+
+        public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)
+        {
+            return workflowCreator.CreateWorkflow(namespaceOverride);
+        }
+
+        public void Decommission(bool deleteKubernetesResources, bool deleteTrackedFiles, bool waitTillDone)
+        {
+            if (deleteKubernetesResources) CreateWorkflow().DeleteNamespace(waitTillDone);
+            if (deleteTrackedFiles) fileManager.DeleteAllFiles();
+        }
+
+        public IFileManager GetFileManager()
+        {
+            return fileManager;
+        }
+
+        public ILog GetLog()
+        {
+            return log;
+        }
+    }
+}

Framework/Core/ProjectPlugin.cs

@@ -0,0 +1,34 @@
+using Utils;
+
+namespace Core
+{
+    public interface IProjectPlugin
+    {
+        void Announce();
+        void Decommission();
+    }
+
+    public interface IHasLogPrefix
+    {
+        string LogPrefix { get; }
+    }
+
+    public interface IHasMetadata
+    {
+        void AddMetadata(IAddMetadata metadata);
+    }
+
+    public static class ProjectPlugin
+    {
+        /// <summary>
+        /// On some platforms and in some cases, not all required plugin assemblies are automatically loaded into the app domain.
+        /// In this case, the runtime needs a slight push to load it before the EntryPoint class is instantiated.
+        /// Used ProjectPlugin.Load<>() before you create an EntryPoint to ensure all plugins you want to use are loaded.
+        /// </summary>
+        public static void Load<T>() where T : IProjectPlugin
+        {
+            var type = typeof(T);
+            FrameworkAssert.That(type != null, $"Unable to load plugin.");
+        }
+    }
+}

Framework/Core/SerializeGate.cs

@@ -0,0 +1,18 @@
+using Newtonsoft.Json;
+
+namespace Core
+{
+    public static class SerializeGate
+    {
+        /// <summary>
+        /// SerializeGate was added to help ensure deployment objects are serializable and remain viable after deserialization.
+        /// Tools can be built on top of the core interface that rely on deployment objects being serializable.
+        /// Insert the serialization gate after deployment but before wrapping to ensure any future changes don't break this requirement.
+        /// </summary>
+        public static T Gate<T>(T anything)
+        {
+            var json = JsonConvert.SerializeObject(anything);
+            return JsonConvert.DeserializeObject<T>(json)!;
+        }
+    }
+}

Framework/Core/TimeSet.cs

@@ -0,0 +1,88 @@
+namespace Core
+{
+    public interface ITimeSet
+    {
+        /// <summary>
+        /// Timeout for a single HTTP call.
+        /// </summary>
+        TimeSpan HttpCallTimeout();
+
+        /// <summary>
+        /// Maximum total time to attempt to make a successful HTTP call to a service.
+        /// When HTTP calls time out during this timespan, retries will be made.
+        /// </summary>
+        TimeSpan HttpRetryTimeout();
+
+        /// <summary>
+        /// After a failed HTTP call, wait this long before trying again.
+        /// </summary>
+        TimeSpan HttpCallRetryDelay();
+
+        /// <summary>
+        /// After a failed K8s operation, wait this long before trying again.
+        /// </summary>
+        TimeSpan K8sOperationRetryDelay();
+
+        /// <summary>
+        /// Maximum total time to attempt to perform a successful k8s operation.
+        /// If k8s operations fail during this timespan, retries will be made.
+        /// </summary>
+        TimeSpan K8sOperationTimeout();
+    }
+
+    public class DefaultTimeSet : ITimeSet
+    {
+        public TimeSpan HttpCallTimeout()
+        {
+            return TimeSpan.FromMinutes(3);
+        }
+
+        public TimeSpan HttpRetryTimeout()
+        {
+            return TimeSpan.FromMinutes(10);
+        }
+
+        public TimeSpan HttpCallRetryDelay()
+        {
+            return TimeSpan.FromSeconds(1);
+        }
+
+        public TimeSpan K8sOperationRetryDelay()
+        {
+            return TimeSpan.FromSeconds(10);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromMinutes(30);
+        }
+    }
+
+    public class LongTimeSet : ITimeSet
+    {
+        public TimeSpan HttpCallTimeout()
+        {
+            return TimeSpan.FromMinutes(30);
+        }
+
+        public TimeSpan HttpRetryTimeout()
+        {
+            return TimeSpan.FromHours(2.2);
+        }
+
+        public TimeSpan HttpCallRetryDelay()
+        {
+            return TimeSpan.FromSeconds(20);
+        }
+
+        public TimeSpan K8sOperationRetryDelay()
+        {
+            return TimeSpan.FromSeconds(30);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromHours(1);
+        }
+    }
+}

Framework/Core/ToolsFactory.cs

@@ -0,0 +1,31 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace Core
+{
+    internal interface IToolsFactory
+    {
+        PluginTools CreateTools();
+    }
+
+    internal class ToolsFactory : IToolsFactory
+    {
+        private readonly ILog log;
+        private readonly WorkflowCreator workflowCreator;
+        private readonly string fileManagerRootFolder;
+        private readonly ITimeSet timeSet;
+
+        public ToolsFactory(ILog log, Configuration configuration, string fileManagerRootFolder, ITimeSet timeSet)
+        {
+            this.log = log;
+            workflowCreator = new WorkflowCreator(log, configuration);
+            this.fileManagerRootFolder = fileManagerRootFolder;
+            this.timeSet = timeSet;
+        }
+
+        public PluginTools CreateTools()
+        {
+            return new PluginTools(log, workflowCreator, fileManagerRootFolder, timeSet);
+        }
+    }
+}

Framework/DiscordRewards/CheckConfig.cs

@@ -0,0 +1,21 @@
+using Utils;
+
+namespace DiscordRewards
+{
+    public class CheckConfig
+    {
+        public CheckType Type { get; set; }
+        public ulong MinNumberOfHosts { get; set; }
+        public ByteSize MinSlotSize { get; set; } = 0.Bytes();
+        public TimeSpan MinDuration { get; set; } = TimeSpan.Zero;
+    }
+
+    public enum CheckType
+    {
+        Uninitialized,
+        HostFilledSlot,
+        HostFinishedSlot,
+        ClientPostedContract,
+        ClientStartedContract,
+    }
+}

Framework/DiscordRewards/DiscordRewards.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -0,0 +1,31 @@
+namespace DiscordRewards
+{
+    public class GiveRewardsCommand
+    {
+        public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
+        public MarketAverage[] Averages { get; set; } = Array.Empty<MarketAverage>();
+        public string[] EventsOverview { get; set; } = Array.Empty<string>();
+
+        public bool HasAny()
+        {
+            return Rewards.Any() || Averages.Any() || EventsOverview.Any();
+        }
+    }
+
+    public class RewardUsersCommand
+    {
+        public ulong RewardId { get; set; }
+        public string[] UserAddresses { get; set; } = Array.Empty<string>();
+    }
+
+    public class MarketAverage
+    {
+        public int NumberOfFinished { get; set; }
+        public int TimeRangeSeconds { get; set; }
+        public float Price { get; set; }
+        public float Size { get; set; }
+        public float Duration { get; set; }
+        public float Collateral { get; set; }
+        public float ProofProbability { get; set; }
+    }
+}

Framework/DiscordRewards/RewardConfig.cs

@@ -0,0 +1,18 @@
+namespace DiscordRewards
+{
+    public class RewardConfig
+    {
+        public const string UsernameTag = "<USER>";
+
+        public RewardConfig(ulong roleId, string message, CheckConfig checkConfig)
+        {
+            RoleId = roleId;
+            Message = message;
+            CheckConfig = checkConfig;
+        }
+
+        public ulong RoleId { get; }
+        public string Message { get; }
+        public CheckConfig CheckConfig { get; }
+    }
+}

Framework/DiscordRewards/RewardRepo.cs

@@ -0,0 +1,53 @@
+namespace DiscordRewards
+{
+    public class RewardRepo
+    {
+        private static string Tag => RewardConfig.UsernameTag;
+
+        public RewardConfig[] Rewards { get; } = new RewardConfig[0];
+
+        // Example configuration, from test server:
+        //{
+        //    // Filled any slot
+        //    new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFilledSlot
+        //    }),
+
+        //    // Finished any slot
+        //    new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFinishedSlot
+        //    }),
+
+        //    // Finished a sizable slot
+        //    new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig
+        //    {
+        //        Type = CheckType.HostFinishedSlot,
+        //        MinSlotSize = 10.MB(),
+        //        MinDuration = TimeSpan.FromMinutes(5.0),
+        //    }),
+
+        //    // Posted any contract
+        //    new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientPostedContract
+        //    }),
+
+        //    // Started any contract
+        //    new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientStartedContract
+        //    }),
+
+        //    // Started a sizable contract
+        //    new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig
+        //    {
+        //        Type = CheckType.ClientStartedContract,
+        //        MinNumberOfHosts = 4,
+        //        MinSlotSize = 10.MB(),
+        //        MinDuration = TimeSpan.FromMinutes(5.0),
+        //    })
+        //};
+    }
+}

Framework/FileUtils/FileManager.cs

@@ -0,0 +1,168 @@
+using Logging;
+using Utils;
+
+namespace FileUtils
+{
+    public interface IFileManager
+    {
+        TrackedFile CreateEmptyFile(string label = "");
+        TrackedFile GenerateFile(ByteSize size, string label = "");
+        void DeleteAllFiles();
+        void ScopedFiles(Action action);
+        T ScopedFiles<T>(Func<T> action);
+    }
+
+    public class FileManager : IFileManager
+    {
+        public const int ChunkSize = 1024 * 1024 * 100;
+        private static NumberSource folderNumberSource = new NumberSource(0);
+        private readonly Random random = new Random();
+        private readonly ILog log;
+        private readonly string rootFolder;
+        private readonly string folder;
+        private readonly List<List<TrackedFile>> fileSetStack = new List<List<TrackedFile>>();
+
+        public FileManager(ILog log, string rootFolder)
+        {
+            folder = Path.Combine(rootFolder, folderNumberSource.GetNextNumber().ToString("D5"));
+
+            this.log = log;
+            this.rootFolder = rootFolder;
+        }
+
+        public TrackedFile CreateEmptyFile(string label = "")
+        {
+            var path = Path.Combine(folder, Guid.NewGuid().ToString() + ".bin");
+            EnsureDirectory();
+
+            var result = new TrackedFile(log, path, label);
+            File.Create(result.Filename).Close();
+            if (fileSetStack.Any()) fileSetStack.Last().Add(result);
+            return result;
+        }
+
+        public TrackedFile GenerateFile(ByteSize size, string label)
+        {
+            var sw = Stopwatch.Begin(log);
+            var result = GenerateRandomFile(size, label);
+            sw.End($"Generated file {result.Describe()}.");
+            return result;
+        }
+
+        public void DeleteAllFiles()
+        {
+            DeleteDirectory();
+        }
+
+        public void ScopedFiles(Action action)
+        {
+            PushFileSet();
+            try
+            {
+                action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
+        }
+
+        public T ScopedFiles<T>(Func<T> action)
+        {
+            PushFileSet();
+            try
+            {
+                return action();
+            }
+            finally
+            {
+                PopFileSet();
+            }
+        }
+
+        private void PushFileSet()
+        {
+            fileSetStack.Add(new List<TrackedFile>());
+        }
+
+        private void PopFileSet()
+        {
+            if (!fileSetStack.Any()) return;
+            var pop = fileSetStack.Last();
+            fileSetStack.Remove(pop);
+
+            foreach (var file in pop)
+            {
+                File.Delete(file.Filename);
+            }
+
+            // If the folder is now empty, delete it too.
+            if (!Directory.GetFiles(folder).Any()) DeleteDirectory();
+        }
+
+        private TrackedFile GenerateRandomFile(ByteSize size, string label)
+        {
+            var result = CreateEmptyFile(label);
+            CheckSpaceAvailable(result, size);
+
+            GenerateFileBytes(result, size);
+            return result;
+        }
+
+        private void CheckSpaceAvailable(TrackedFile testFile, ByteSize size)
+        {
+            var file = new FileInfo(testFile.Filename);
+            var drive = new DriveInfo(file.Directory!.Root.FullName);
+
+            var spaceAvailable = drive.TotalFreeSpace;
+
+            if (spaceAvailable < size.SizeInBytes)
+            {
+                var msg = $"Not enough disk space. " +
+                    $"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
+                    $"{Formatter.FormatByteSize(spaceAvailable)} available.";
+
+                log.Log(msg);
+                throw new Exception(msg);
+            }
+        }
+
+        private void GenerateFileBytes(TrackedFile result, ByteSize size)
+        {
+            long bytesLeft = size.SizeInBytes;
+            int chunkSize = ChunkSize;
+            while (bytesLeft > 0)
+            {
+                try
+                {
+                    var length = Math.Min(bytesLeft, chunkSize);
+                    AppendRandomBytesToFile(result, length);
+                    bytesLeft -= length;
+                }
+                catch
+                {
+                    chunkSize = chunkSize / 2;
+                    if (chunkSize < 1024) throw;
+                }
+            }
+        }
+
+        private void AppendRandomBytesToFile(TrackedFile result, long length)
+        {
+            var bytes = new byte[length];
+            random.NextBytes(bytes);
+            using var stream = new FileStream(result.Filename, FileMode.Append);
+            stream.Write(bytes, 0, bytes.Length);
+        }
+
+        private void EnsureDirectory()
+        {
+            Directory.CreateDirectory(folder);
+        }
+
+        private void DeleteDirectory()
+        {
+            if (Directory.Exists(folder)) Directory.Delete(folder, true);
+        }
+    }
+}

Framework/FileUtils/FileUtils.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+    <ProjectReference Include="..\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/FileUtils/TrackedFile.cs

@@ -0,0 +1,87 @@
+using Logging;
+using Utils;
+
+namespace FileUtils
+{
+    public class TrackedFile
+    {
+        private readonly ILog log;
+
+        public TrackedFile(ILog log, string filename, string label)
+        {
+            this.log = log;
+            Filename = filename;
+            Label = label;
+        }
+
+        public string Filename { get; }
+        public string Label { get; }
+
+        public void AssertIsEqual(TrackedFile? actual)
+        {
+            var sw = Stopwatch.Begin(log);
+            try
+            {
+                AssertEqual(actual);
+            }
+            finally
+            {
+                sw.End($"{nameof(TrackedFile)}.{nameof(AssertIsEqual)}");
+            }
+        }
+
+        public string Describe()
+        {
+            var sizePostfix = $" ({Formatter.FormatByteSize(GetFileSize())})";
+            if (!string.IsNullOrEmpty(Label)) return Label + sizePostfix;
+            return $"'{Filename}'{sizePostfix}";
+        }
+
+        public ByteSize GetFilesize()
+        {
+            return new ByteSize(GetFileSize());
+        }
+
+        private void AssertEqual(TrackedFile? actual)
+        {
+            if (actual == null)  FrameworkAssert.Fail("TestFile is null.");
+            if (actual == this || actual!.Filename == Filename) FrameworkAssert.Fail("TestFile is compared to itself.");
+
+            FrameworkAssert.That(actual.GetFileSize() == GetFileSize(), "Files are not of equal length.");
+
+            using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
+            using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
+
+            var bytesExpected = new byte[FileManager.ChunkSize];
+            var bytesActual = new byte[FileManager.ChunkSize];
+
+            var readExpected = 0;
+            var readActual = 0;
+
+            while (true)
+            {
+                readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
+                readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
+
+                if (readExpected == 0 && readActual == 0)
+                {
+                    log.Log($"OK: {Describe()} is equal to {actual.Describe()}.");
+                    return;
+                }
+
+                FrameworkAssert.That(readActual == readExpected, "Unable to read buffers of equal length.");
+
+                for (var i = 0; i < readActual; i++)
+                {
+                    if (bytesExpected[i] != bytesActual[i]) FrameworkAssert.Fail("File contents not equal.");
+                }
+            }
+        }
+
+        private long GetFileSize()
+        {
+            var info = new FileInfo(Filename);
+            return info.Length;
+        }
+    }
+}

Framework/GethConnector/GethConnector.cs

@@ -0,0 +1,39 @@
+using CodexContractsPlugin;
+using GethPlugin;
+using Logging;
+
+namespace GethConnector
+{
+    public class GethConnector
+    {
+        public IGethNode GethNode { get; }
+        public ICodexContracts CodexContracts { get; }
+
+        public static GethConnector? Initialize(ILog log)
+        {
+            if (!string.IsNullOrEmpty(GethInput.LoadError))
+            {
+                var msg = "Geth input incorrect: " + GethInput.LoadError;
+                log.Error(msg);
+                return null;
+            }
+
+            var contractsDeployment = new CodexContractsDeployment(
+                marketplaceAddress: GethInput.MarketplaceAddress,
+                abi: GethInput.ABI,
+                tokenAddress: GethInput.TokenAddress
+            );
+
+            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+            var contracts = new CodexContractsAccess(log, gethNode, contractsDeployment);
+
+            return new GethConnector(gethNode, contracts);
+        }
+
+        private GethConnector(IGethNode gethNode, ICodexContracts codexContracts)
+        {
+            GethNode = gethNode;
+            CodexContracts = codexContracts;
+        }
+    }
+}

Framework/GethConnector/GethConnector.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\GethPlugin\GethPlugin.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/GethConnector/GethInput.cs

@@ -0,0 +1,52 @@
+namespace GethConnector
+{
+    public static class GethInput
+    {
+        private const string GethHostVar = "GETH_HOST";
+        private const string GethPortVar = "GETH_HTTP_PORT";
+        private const string GethPrivKeyVar = "GETH_PRIVATE_KEY";
+        private const string MarketplaceAddressVar = "CODEXCONTRACTS_MARKETPLACEADDRESS";
+        private const string TokenAddressVar = "CODEXCONTRACTS_TOKENADDRESS";
+        private const string AbiVar = "CODEXCONTRACTS_ABI";
+
+        static GethInput()
+        {
+            var error = new List<string>();
+            var gethHost = GetEnvVar(error, GethHostVar);
+            var gethPort = Convert.ToInt32(GetEnvVar(error, GethPortVar));
+            var privateKey = GetEnvVar(error, GethPrivKeyVar);
+            var marketplaceAddress = GetEnvVar(error, MarketplaceAddressVar);
+            var tokenAddress = GetEnvVar(error, TokenAddressVar);
+            var abi = GetEnvVar(error, AbiVar);
+
+            if (error.Any())
+            {
+                LoadError = string.Join(", ", error);
+            }
+            else
+            {
+                GethHost = gethHost!;
+                GethPort = gethPort;
+                PrivateKey = privateKey!;
+                MarketplaceAddress = marketplaceAddress!;
+                TokenAddress = tokenAddress!;
+                ABI = abi!;
+            }
+        }
+
+        public static string GethHost { get; } = string.Empty;
+        public static int GethPort { get; }
+        public static string PrivateKey { get; } = string.Empty;
+        public static string MarketplaceAddress { get; } = string.Empty;
+        public static string TokenAddress { get; } = string.Empty;
+        public static string ABI { get; } = string.Empty;
+        public static string LoadError { get; } = string.Empty;
+
+        private static string? GetEnvVar(List<string> error, string name)
+        {
+            var result = Environment.GetEnvironmentVariable(name);
+            if (string.IsNullOrEmpty(result)) error.Add($"'{name}' is not set.");
+            return result;
+        }
+    }
+}

Framework/KubernetesWorkflow/CommandRunner.cs

@@ -1,23 +1,22 @@
-using k8s;
-using Utils;
+using Utils;
 
 namespace KubernetesWorkflow
 {
     public class CommandRunner
     {
-        private readonly Kubernetes client;
+        private readonly K8sClient client;
         private readonly string k8sNamespace;
-        private readonly RunningPod pod;
+        private readonly string podName;
         private readonly string containerName;
         private readonly string command;
         private readonly string[] arguments;
         private readonly List<string> lines = new List<string>();
 
-        public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
+        public CommandRunner(K8sClient client, string k8sNamespace, string podName, string containerName, string command, string[] arguments)
         {
             this.client = client;
             this.k8sNamespace = k8sNamespace;
-            this.pod = pod;
+            this.podName = podName;
             this.containerName = containerName;
             this.command = command;
             this.arguments = arguments;
@@ -27,8 +26,8 @@ namespace KubernetesWorkflow
         {
             var input = new[] { command }.Concat(arguments).ToArray();
 
-            Time.Wait(client.NamespacedPodExecAsync(
-                pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()));
+            Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
+                podName, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
         }
 
         public string GetStdOut()

Framework/KubernetesWorkflow/Configuration.cs

@@ -0,0 +1,25 @@
+using Newtonsoft.Json;
+
+namespace KubernetesWorkflow
+{
+    public class Configuration
+    {
+        public Configuration(string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, string kubernetesNamespace)
+        {
+            KubeConfigFile = kubeConfigFile;
+            OperationTimeout = operationTimeout;
+            RetryDelay = retryDelay;
+            KubernetesNamespace = kubernetesNamespace;
+        }
+
+        public string? KubeConfigFile { get; }
+        public TimeSpan OperationTimeout { get; }
+        public TimeSpan RetryDelay { get; }
+        public string KubernetesNamespace { get; }
+        public bool AllowNamespaceOverride { get; set; } = true;
+        public bool AddAppPodLabel { get; set; } = true;
+
+        [JsonIgnore]
+        public IK8sHooks Hooks { get; set; } = new DoNothingK8sHooks();
+    }
+}

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -0,0 +1,98 @@
+using k8s;
+using Logging;
+
+namespace KubernetesWorkflow
+{
+    public class CrashWatcher
+    {
+        private readonly ILog log;
+        private readonly KubernetesClientConfiguration config;
+        private readonly string containerName;
+        private readonly string podName;
+        private readonly string recipeName;
+        private readonly string k8sNamespace;
+        private CancellationTokenSource cts;
+        private Task? worker;
+        private Exception? workerException;
+
+        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
+        {
+            this.log = log;
+            this.config = config;
+            this.containerName = containerName;
+            this.podName = podName;
+            this.recipeName = recipeName;
+            this.k8sNamespace = k8sNamespace;
+            cts = new CancellationTokenSource();
+        }
+
+        public void Start()
+        {
+            if (worker != null) throw new InvalidOperationException();
+
+            cts = new CancellationTokenSource();
+            worker = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            if (worker == null) throw new InvalidOperationException();
+
+            cts.Cancel();
+            worker.Wait();
+            worker = null;
+
+            if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
+        }
+
+        public bool HasContainerCrashed()
+        {
+            using var client = new Kubernetes(config);
+            var result = HasContainerBeenRestarted(client);
+            if (result) DownloadCrashedContainerLogs(client);
+            return result;
+        }
+
+        private void Worker()
+        {
+            try
+            {
+                MonitorContainer(cts.Token);
+            }
+            catch (Exception ex)
+            {
+                workerException = ex;
+            }
+        }
+
+        private void MonitorContainer(CancellationToken token)
+        {
+            using var client = new Kubernetes(config);
+            while (!token.IsCancellationRequested)
+            {
+                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10));
+
+                if (HasContainerBeenRestarted(client))
+                {
+                    DownloadCrashedContainerLogs(client);
+                    return;
+                }
+            }
+        }
+
+        private bool HasContainerBeenRestarted(Kubernetes client)
+        {
+            var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
+            var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            if (result) log.Log("Pod crash detected for " + containerName);
+            return result;
+        }
+
+        private void DownloadCrashedContainerLogs(Kubernetes client)
+        {
+            using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
+            var handler = new WriteToFileLogHandler(log, "Crash detected for " + containerName);
+            handler.Log(stream);
+        }
+    }
+}

Framework/KubernetesWorkflow/K8sClient.cs

@@ -0,0 +1,36 @@
+using k8s;
+
+namespace KubernetesWorkflow
+{
+    public class K8sClient
+    {
+        private readonly Kubernetes client;
+        private static readonly object clientLock = new object();
+
+        public K8sClient(KubernetesClientConfiguration config)
+        {
+            client = new Kubernetes(config);
+        }
+
+        public void Run(Action<Kubernetes> action)
+        {
+            lock (clientLock)
+            {
+                action(client);
+            }
+        }
+
+        public T Run<T>(Func<Kubernetes, T> action)
+        {
+            lock (clientLock)
+            {
+                return action(client);
+            }
+        }
+
+        public void Dispose()
+        {
+            client.Dispose();
+        }
+    }
+}

Framework/KubernetesWorkflow/K8sCluster.cs

@@ -8,29 +8,24 @@ namespace KubernetesWorkflow
         {
             Configuration = configuration;
         }
-       
+
         public Configuration Configuration { get; }
-        public string IP { get; private set; } = string.Empty;
+        public string HostAddress { get; private set; } = string.Empty;
 
         public KubernetesClientConfiguration GetK8sClientConfig()
         {
             var config = GetConfig();
-            UpdateIp(config);
+            UpdateHostAddress(config);
+            config.SkipTlsVerify = true; // Required for operation on Wings cluster.
             return config;
         }
 
-        public string GetNodeLabelForLocation(Location location)
-        {
-            if (location == Location.Unspecified) return string.Empty;
-            return Configuration.LocationMap.Single(l => l.Location == location).WorkerName;
-        }
-
         public TimeSpan K8sOperationTimeout()
         {
             return Configuration.OperationTimeout;
         }
 
-        public TimeSpan WaitForK8sServiceDelay()
+        public TimeSpan K8sOperationRetryDelay()
         {
             return Configuration.RetryDelay;
         }
@@ -47,10 +42,17 @@ namespace KubernetesWorkflow
             }
         }
 
-        private void UpdateIp(KubernetesClientConfiguration config)
+        private void UpdateHostAddress(KubernetesClientConfiguration config)
         {
             var host = config.Host.Replace("https://", "");
-            IP = host.Substring(0, host.IndexOf(':'));
+            if (host.Contains(":"))
+            {
+                HostAddress = "http://" + host.Substring(0, host.IndexOf(':'));
+            }
+            else
+            {
+                HostAddress = config.Host;
+            }
         }
     }
 }

Framework/KubernetesWorkflow/K8sController.cs

@@ -0,0 +1,976 @@
+using k8s;
+using k8s.Models;
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
+using Logging;
+using Utils;
+
+namespace KubernetesWorkflow
+{
+    public class K8sController
+    {
+        private readonly ILog log;
+        private readonly K8sCluster cluster;
+        private readonly WorkflowNumberSource workflowNumberSource;
+        private readonly K8sClient client;
+        public const string PodLabelKey = "pod-uuid";
+
+        public K8sController(ILog log, K8sCluster cluster, WorkflowNumberSource workflowNumberSource, string k8sNamespace)
+        {
+            this.log = log;
+            this.cluster = cluster;
+            this.workflowNumberSource = workflowNumberSource;
+            client = new K8sClient(cluster.GetK8sClientConfig());
+
+            K8sNamespace = k8sNamespace;
+        }
+
+        public void Dispose()
+        {
+            client.Dispose();
+        }
+
+        public StartResult BringOnline(ContainerRecipe[] containerRecipes, ILocation location)
+        {
+            log.Debug();
+            EnsureNamespace();
+
+            var podLabel = K8sNameUtils.Format(Guid.NewGuid().ToString());
+            var deployment = CreateDeployment(containerRecipes, location, podLabel);
+            var internalService = CreateInternalService(containerRecipes);
+            var externalService = CreateExternalService(containerRecipes);
+
+            return new StartResult(cluster, containerRecipes, deployment, internalService, externalService);
+        }
+
+        public void WaitUntilOnline(RunningContainer container)
+        {
+            WaitUntilDeploymentOnline(container);
+        }
+
+        public PodInfo GetPodInfo(RunningDeployment deployment)
+        {
+            var pod = GetPodForDeployment(deployment);
+            return CreatePodInfo(pod);
+        }
+
+        public void Stop(StartResult startResult, bool waitTillStopped)
+        {
+            log.Debug();
+            if (startResult.InternalService != null) DeleteService(startResult.InternalService);
+            if (startResult.ExternalService != null) DeleteService(startResult.ExternalService);
+            DeleteDeployment(startResult.Deployment);
+
+            if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
+        }
+
+        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines, bool? previous)
+        {
+            log.Debug();
+
+            var podName = GetPodName(container);
+            var recipeName = container.Recipe.Name;
+
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines, previous: previous));
+            logHandler.Log(stream);
+        }
+
+        public string ExecuteCommand(RunningContainer container, string command, params string[] args)
+        {
+            var containerName = container.Recipe.Name;
+            var cmdAndArgs = $"{containerName}: {command} ({string.Join(",", args)})";
+            log.Debug(cmdAndArgs);
+
+            var podName = GetPodName(container);
+            var runner = new CommandRunner(client, K8sNamespace, podName, containerName, command, args);
+            runner.Run();
+            var result = runner.GetStdOut();
+
+            log.Debug($"{cmdAndArgs} = '{result}'");
+            return result;
+        }
+
+        public int[] GetUsedExternalPorts()
+        {
+            return client.Run(c =>
+            {
+                var result = new List<int>();
+
+                var services = c.ListServiceForAllNamespaces();
+                var nodePorts = services.Items.Where(s => s.Spec.Type == "NodePort").ToArray();
+                if (!nodePorts.Any()) return result.ToArray();
+
+                foreach (var service in nodePorts)
+                {
+                    foreach (var port in service.Spec.Ports)
+                    {
+                        if (port.NodePort.HasValue)
+                        {
+                            result.Add(port.NodePort.Value);
+                        }
+                    }
+                }
+
+                return result.ToArray();
+            });
+        }
+
+        public void DeleteAllNamespacesStartingWith(string prefix, bool wait)
+        {
+            log.Debug();
+
+            var all = client.Run(c => c.ListNamespace().Items);
+            var namespaces = all.Select(n => n.Name()).Where(n => n.StartsWith(prefix));
+
+            foreach (var ns in namespaces)
+            {
+                DeleteNamespace(ns, wait);
+            }
+        }
+
+        public void DeleteNamespace(bool wait)
+        {
+            log.Debug();
+            if (IsNamespaceOnline(K8sNamespace))
+            {
+                client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
+
+                if (wait) WaitUntilNamespaceDeleted(K8sNamespace);
+            }
+        }
+
+        public void DeleteNamespace(string ns, bool wait)
+        {
+            log.Debug();
+            if (IsNamespaceOnline(ns))
+            {
+                client.Run(c => c.DeleteNamespace(ns, null, null, gracePeriodSeconds: 0));
+                if (wait) WaitUntilNamespaceDeleted(ns);
+            }
+        }
+
+        #region Discover K8s Nodes
+
+        public K8sNodeLabel[] GetAvailableK8sNodes()
+        {
+            var nodes = client.Run(c => c.ListNode());
+
+            var optionals = nodes.Items.Select(i => CreateNodeLabel(i));
+            return optionals.Where(n => n != null).Select(n => n!).ToArray();
+        }
+
+        private K8sNodeLabel? CreateNodeLabel(V1Node i)
+        {
+            var keys = i.Metadata.Labels.Keys;
+            var hostnameKey = keys.SingleOrDefault(k => k.ToLowerInvariant().Contains("hostname"));
+            if (hostnameKey != null)
+            {
+                var hostnameValue = i.Metadata.Labels[hostnameKey];
+                return new K8sNodeLabel(hostnameKey, hostnameValue);
+            }
+            return null;
+        }
+
+        #endregion
+
+        #region Namespace management
+
+        private string K8sNamespace { get; }
+
+        private void EnsureNamespace()
+        {
+            if (IsNamespaceOnline(K8sNamespace)) return;
+
+            var namespaceSpec = new V1Namespace
+            {
+                ApiVersion = "v1",
+                Metadata = new V1ObjectMeta
+                {
+                    Name = K8sNamespace,
+                    Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
+                }
+            };
+            client.Run(c => c.CreateNamespace(namespaceSpec));
+            WaitUntilNamespaceCreated();
+
+            CreatePolicy();
+        }
+
+        private bool IsNamespaceOnline(string name)
+        {
+            return client.Run(c => c.ListNamespace().Items.Any(n => n.Metadata.Name == name));
+        }
+
+        private void CreatePolicy()
+        {
+            client.Run(c =>
+            {
+                var body = new V1NetworkPolicy
+                {
+                    Metadata = new V1ObjectMeta
+                    {
+                        Name = "isolate-policy",
+                        NamespaceProperty = K8sNamespace
+                    },
+                    Spec = new V1NetworkPolicySpec
+                    {
+                        PodSelector = new V1LabelSelector {},
+                        PolicyTypes = new[]
+                        {
+                            "Ingress",
+                            "Egress"
+                        },
+                        Ingress = new List<V1NetworkPolicyIngressRule>
+                        {
+                            new V1NetworkPolicyIngressRule
+                            {
+                                FromProperty = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        PodSelector = new V1LabelSelector {}
+                                    }
+                                }
+                            },
+                            new V1NetworkPolicyIngressRule
+                            {
+                                FromProperty = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        NamespaceSelector = new V1LabelSelector
+                                        {
+                                            MatchLabels = GetRunnerNamespaceSelector()
+                                        }
+                                    }
+                                }
+                            },
+                            new V1NetworkPolicyIngressRule
+                            {
+                                FromProperty = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        NamespaceSelector = new V1LabelSelector
+                                        {
+                                            MatchLabels = GetPrometheusNamespaceSelector()
+                                        }
+                                    }
+                                }
+                            }
+                        },
+                        Egress = new List<V1NetworkPolicyEgressRule>
+                        {
+                            new V1NetworkPolicyEgressRule
+                            {
+                                To = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        PodSelector = new V1LabelSelector {}
+                                    }
+                                }
+                            },
+                            new V1NetworkPolicyEgressRule
+                            {
+                                To = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        NamespaceSelector = new V1LabelSelector
+                                        {
+                                            MatchLabels = new Dictionary<string, string> { { "kubernetes.io/metadata.name", "kube-system" } }
+                                        }
+                                    },
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        PodSelector = new V1LabelSelector
+                                        {
+                                            MatchLabels = new Dictionary<string, string> { { "k8s-app", "kube-dns" } }
+                                        }
+                                    }
+                                },
+                                Ports = new List<V1NetworkPolicyPort>
+                                {
+                                    new V1NetworkPolicyPort
+                                    {
+                                        Port = new IntstrIntOrString
+                                        {
+                                            Value = "53"
+                                        },
+                                        Protocol = "UDP"
+                                    }
+                                }
+                            },
+                            new V1NetworkPolicyEgressRule
+                            {
+                                To = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        IpBlock = new V1IPBlock
+                                        {
+                                            Cidr = "0.0.0.0/0"
+                                        }
+                                    }
+                                },
+                                Ports = new List<V1NetworkPolicyPort>
+                                {
+                                    new V1NetworkPolicyPort
+                                    {
+                                        Port = new IntstrIntOrString
+                                        {
+                                            Value = "80"
+                                        },
+                                        Protocol = "TCP"
+                                    },
+                                    new V1NetworkPolicyPort
+                                    {
+                                        Port = new IntstrIntOrString
+                                        {
+                                            Value = "443"
+                                        },
+                                        Protocol = "TCP"
+                                    }
+                                }
+                            }
+
+                        }
+                    }
+                };
+
+                c.CreateNamespacedNetworkPolicy(body, K8sNamespace);
+            });
+        }
+
+        #endregion
+
+        #region Deployment management
+
+        private RunningDeployment CreateDeployment(ContainerRecipe[] containerRecipes, ILocation location, string podLabel)
+        {
+            var deploymentSpec = new V1Deployment
+            {
+                ApiVersion = "apps/v1",
+                Metadata = CreateDeploymentMetadata(containerRecipes),
+                Spec = new V1DeploymentSpec
+                {
+                    Replicas = 1,
+                    Selector = new V1LabelSelector
+                    {
+                        MatchLabels = GetSelector(containerRecipes)
+                    },
+                    Template = new V1PodTemplateSpec
+                    {
+                        Metadata = new V1ObjectMeta
+                        {
+                            Labels = GetSelector(containerRecipes, podLabel),
+                            Annotations = GetAnnotations(containerRecipes)
+                        },
+                        Spec = new V1PodSpec
+                        {
+                            PriorityClassName = GetPriorityClassName(containerRecipes),
+                            Affinity = CreatePodAffinity(containerRecipes),
+                            NodeSelector = CreateNodeSelector(location),
+                            Containers = CreateDeploymentContainers(containerRecipes),
+                            Volumes = CreateVolumes(containerRecipes)
+                        }
+                    }
+                }
+            };
+
+            client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
+
+            var name = deploymentSpec.Metadata.Name;
+            return new RunningDeployment(name, podLabel);
+        }
+
+        private void DeleteDeployment(RunningDeployment deployment)
+        {
+            client.Run(c => c.DeleteNamespacedDeployment(deployment.Name, K8sNamespace));
+            WaitUntilDeploymentOffline(deployment.Name);
+        }
+
+        private IDictionary<string, string> CreateNodeSelector(ILocation location)
+        {
+            var nodeLabel = GetNodeLabelForLocation(location);
+            if (nodeLabel == null) return new Dictionary<string, string>();
+
+            return new Dictionary<string, string>
+            {
+                { nodeLabel.Key, nodeLabel.Value }
+            };
+        }
+
+        private V1Affinity? CreatePodAffinity(ContainerRecipe[] recipes)
+        {
+            var notIns = recipes
+                .Select(r => r.SchedulingAffinity.NotIn)
+                .Where(n => !string.IsNullOrEmpty(n))
+                .Distinct()
+                .ToList();
+
+            if (!notIns.Any()) return null;
+
+            return new V1Affinity
+            {
+                NodeAffinity = new V1NodeAffinity
+                {
+                    RequiredDuringSchedulingIgnoredDuringExecution = new V1NodeSelector
+                    {
+                        NodeSelectorTerms = new List<V1NodeSelectorTerm>
+                        { 
+                            new V1NodeSelectorTerm
+                            {
+                                MatchExpressions = new List<V1NodeSelectorRequirement>
+                                {
+                                    new V1NodeSelectorRequirement
+                                    {
+                                        Key = "allow-tests-pods",
+                                        OperatorProperty = "NotIn",
+                                        Values = notIns
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            };
+        }
+
+        private K8sNodeLabel? GetNodeLabelForLocation(ILocation location)
+        {
+            var l = (Location)location;
+            return l.NodeLabel;
+        }
+
+        private string GetPriorityClassName(ContainerRecipe[] containerRecipes)
+        {
+            if (containerRecipes.Any(c => c.SetCriticalPriority))
+            {
+                return "system-node-critical";
+            }
+            return null!;
+        }
+
+        private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes)
+        {
+            return containerRecipes.First().PodLabels.GetLabels();
+        }
+
+        private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes, string podLabel)
+        {
+            var labels = containerRecipes.First().PodLabels.Clone();
+            labels.Add(PodLabelKey, podLabel);
+            return labels.GetLabels();
+        }
+
+        private IDictionary<string, string> GetRunnerNamespaceSelector()
+        {
+            return new Dictionary<string, string> { { "kubernetes.io/metadata.name", "default" } };
+        }
+
+        private IDictionary<string, string> GetPrometheusNamespaceSelector()
+        {
+            return new Dictionary<string, string> { { "kubernetes.io/metadata.name", "monitoring" } };
+        }
+
+        private IDictionary<string, string> GetAnnotations(ContainerRecipe[] containerRecipes)
+        {
+            return containerRecipes.First().PodAnnotations.GetAnnotations();
+        }
+
+        private V1ObjectMeta CreateDeploymentMetadata(ContainerRecipe[] containerRecipes)
+        {
+            return new V1ObjectMeta
+            {
+                Name = string.Join('-',containerRecipes.Select(r => r.Name)),
+                NamespaceProperty = K8sNamespace,
+                Labels = GetSelector(containerRecipes),
+                Annotations = GetAnnotations(containerRecipes)
+            };
+        }
+
+        private List<V1Container> CreateDeploymentContainers(ContainerRecipe[] containerRecipes)
+        {
+            return containerRecipes.Select(CreateDeploymentContainer).ToList();
+        }
+
+        private V1Container CreateDeploymentContainer(ContainerRecipe recipe)
+        {
+            return new V1Container
+            {
+                Name = recipe.Name,
+                Image = recipe.Image,
+                ImagePullPolicy = "Always",
+                Ports = CreateContainerPorts(recipe),
+                Env = CreateEnv(recipe),
+                VolumeMounts = CreateContainerVolumeMounts(recipe),
+                Resources = CreateResourceLimits(recipe),
+                Command = CreateCommandList(recipe)
+            };
+        }
+
+        private IList<string> CreateCommandList(ContainerRecipe recipe)
+        {
+            if (recipe.CommandOverride == null || !recipe.CommandOverride.Command.Any()) return null!;
+            return recipe.CommandOverride.Command.ToList();
+        }
+
+        private V1ResourceRequirements CreateResourceLimits(ContainerRecipe recipe)
+        {
+            return new V1ResourceRequirements
+            {
+                Requests = CreateResourceQuantities(recipe.Resources.Requests),
+                Limits = CreateResourceQuantities(recipe.Resources.Limits)
+            };
+        }
+
+        private Dictionary<string, ResourceQuantity> CreateResourceQuantities(ContainerResourceSet set)
+        {
+            var result = new Dictionary<string, ResourceQuantity>();
+            if (set.MilliCPUs != 0)
+            {
+                result.Add("cpu", new ResourceQuantity($"{set.MilliCPUs}m"));
+            }
+            if (set.Memory.SizeInBytes != 0)
+            {
+                result.Add("memory", new ResourceQuantity(set.Memory.SizeInBytes.ToString()));
+            }
+            return result;
+        }
+
+        private List<V1VolumeMount> CreateContainerVolumeMounts(ContainerRecipe recipe)
+        {
+            return recipe.Volumes.Select(CreateContainerVolumeMount).ToList();
+        }
+
+        private V1VolumeMount CreateContainerVolumeMount(VolumeMount v)
+        {
+            return new V1VolumeMount
+            {
+                Name = v.VolumeName,
+                MountPath = v.MountPath,
+                SubPath = v.SubPath,
+            };
+        }
+
+        private List<V1Volume> CreateVolumes(ContainerRecipe[] containerRecipes)
+        {
+            return containerRecipes.Where(c => c.Volumes.Any()).SelectMany(CreateVolumes).ToList();
+        }
+
+        private List<V1Volume> CreateVolumes(ContainerRecipe recipe)
+        {
+            return recipe.Volumes.Select(CreateVolume).ToList();
+        }
+
+        private V1Volume CreateVolume(VolumeMount v)
+        {
+            CreatePersistentVolumeClaimIfNeeded(v);
+
+            if (!string.IsNullOrEmpty(v.HostPath))
+            {
+                return new V1Volume
+                {
+                    Name = v.VolumeName,
+                    HostPath = new V1HostPathVolumeSource
+                    {
+                        Path = v.HostPath
+                    }
+                };
+            }
+
+            if (!string.IsNullOrEmpty(v.Secret))
+            {
+                return new V1Volume
+                {
+                    Name = v.VolumeName,
+                    Secret = CreateVolumeSecret(v)
+                };
+            }
+
+            return new V1Volume
+            {
+                Name = v.VolumeName,
+                PersistentVolumeClaim = new V1PersistentVolumeClaimVolumeSource
+                {
+                    ClaimName = v.VolumeName
+                }
+            };
+        }
+
+        private void CreatePersistentVolumeClaimIfNeeded(VolumeMount v)
+        {
+            var pvcs = client.Run(c => c.ListNamespacedPersistentVolumeClaim(K8sNamespace));
+            if (pvcs != null && pvcs.Items.Any(i => i.Name() == v.VolumeName)) return;
+
+            client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim
+            {
+                ApiVersion = "v1",
+                Metadata = new V1ObjectMeta
+                {
+                    Name = v.VolumeName,
+                },
+                Spec = new V1PersistentVolumeClaimSpec
+                {
+                    AccessModes = new List<string>
+                    {
+                        "ReadWriteOnce"
+                    },
+                    Resources = CreateVolumeResourceRequirements(v),
+                },
+            }, K8sNamespace));
+        }
+
+        private V1SecretVolumeSource CreateVolumeSecret(VolumeMount v)
+        {
+            if (string.IsNullOrWhiteSpace(v.Secret)) return null!;
+            return new V1SecretVolumeSource
+            {
+                SecretName = v.Secret
+            };
+        }
+
+        private V1ResourceRequirements CreateVolumeResourceRequirements(VolumeMount v)
+        {
+            if (v.ResourceQuantity == null) return null!;
+            return new V1ResourceRequirements
+            {
+                Requests = new Dictionary<string, ResourceQuantity>()
+                {
+                    {"storage", new ResourceQuantity(v.ResourceQuantity) }
+                }
+            };
+        }
+
+        private List<V1EnvVar> CreateEnv(ContainerRecipe recipe)
+        {
+            return recipe.EnvVars.Select(CreateEnvVar).ToList();
+        }
+
+        private V1EnvVar CreateEnvVar(EnvVar envVar)
+        {
+            return new V1EnvVar
+            {
+                Name = envVar.Name,
+                Value = envVar.Value,
+            };
+        }
+
+        private List<V1ContainerPort> CreateContainerPorts(ContainerRecipe recipe)
+        {
+            var exposedPorts = recipe.ExposedPorts.SelectMany(p => CreateContainerPort(recipe, p));
+            var internalPorts = recipe.InternalPorts.SelectMany(p => CreateContainerPort(recipe, p));
+            return exposedPorts.Concat(internalPorts).ToList();
+        }
+
+        private List<V1ContainerPort> CreateContainerPort(ContainerRecipe recipe, Port port)
+        {
+            var result = new List<V1ContainerPort>();
+            if (port.IsTcp()) CreateTcpContainerPort(result, recipe, port);
+            if (port.IsUdp()) CreateUdpContainerPort(result, recipe, port);
+            return result;
+        }
+
+        private void CreateUdpContainerPort(List<V1ContainerPort> result, ContainerRecipe recipe, Port port)
+        {
+            result.Add(CreateContainerPort(recipe, port, "UDP"));
+        }
+
+        private void CreateTcpContainerPort(List<V1ContainerPort> result, ContainerRecipe recipe, Port port)
+        {
+            result.Add(CreateContainerPort(recipe, port, "TCP"));
+        }
+
+        private V1ContainerPort CreateContainerPort(ContainerRecipe recipe, Port port, string protocol)
+        {
+            return new V1ContainerPort
+            {
+                Name = GetNameForPort(recipe, port),
+                ContainerPort = port.Number,
+                Protocol = protocol
+            };
+        }
+
+        private string GetNameForPort(ContainerRecipe recipe, Port port)
+        {
+            var inputs = new[]
+            {
+                $"p{workflowNumberSource.WorkflowNumber}",
+                recipe.Number.ToString(),
+                port.Number.ToString(),
+                port.Protocol.ToString().ToLowerInvariant()
+            };
+
+            return K8sNameUtils.FormatPortName(string.Join(",", inputs));
+        }
+
+        private string GetPodName(RunningContainer container)
+        {
+            return GetPodForDeployment(container.RunningPod.StartResult.Deployment).Metadata.Name;
+        }
+
+        private V1Pod GetPodForDeployment(RunningDeployment deployment)
+        {
+            return Time.Retry(() => GetPodForDeplomentInternal(deployment),
+                // We will wait up to 1 minute, k8s might be moving pods around.
+                maxTimeout: TimeSpan.FromMinutes(1),
+                retryTime: TimeSpan.FromSeconds(10),
+                description: "Find pod by label for deployment.");
+        }
+
+        private V1Pod GetPodForDeplomentInternal(RunningDeployment deployment)
+        {
+            var allPods = client.Run(c => c.ListNamespacedPod(K8sNamespace));
+            var pods = allPods.Items.Where(p => p.GetLabel(PodLabelKey) == deployment.PodLabel).ToArray();
+
+            if (pods.Length != 1)
+            {
+                var allLabels = allPods.Items.Select(p =>
+                {
+                    var labels = string.Join(",", p.Labels().Select(l => $"{l.Key}={l.Value}"));
+                    return $"pod:'{p.Name()}' has labels: [{labels}]";
+                });
+                throw new Exception($"Expected to find 1 pod by podLabel '{deployment.PodLabel}'. Found: {pods.Length}. " +
+                    $"Total number of pods: {allPods.Items.Count}. Their labels: {string.Join(Environment.NewLine, allLabels)}");
+            }
+            return pods[0];
+        }
+
+        #endregion
+
+        #region Service management
+
+        private RunningService? CreateInternalService(ContainerRecipe[] recipes)
+        {
+            return CreateService(recipes, r => r.InternalPorts.Concat(r.ExposedPorts).ToArray(), "ClusterIP", "int", false);
+        }
+
+        private RunningService? CreateExternalService(ContainerRecipe[] recipes)
+        {
+            return CreateService(recipes, r => r.ExposedPorts, "NodePort", "ext", true);
+        }
+
+        private RunningService? CreateService(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector, string serviceType, string namePostfix, bool isNodePort)
+        {
+            var ports = CreateServicePorts(recipes, portSelector, isNodePort);
+            if (!ports.Any()) return null;
+
+            var serviceSpec = new V1Service
+            {
+                ApiVersion = "v1",
+                Metadata = CreateServiceMetadata(recipes, namePostfix),
+                Spec = new V1ServiceSpec
+                {
+                    Type = serviceType,
+                    Selector = GetSelector(recipes),
+                    Ports = ports,
+                }
+            };
+
+            client.Run(c => c.CreateNamespacedService(serviceSpec, K8sNamespace));
+
+            var result = ReadBackServiceAndMapPorts(serviceSpec, recipes);
+            var name = serviceSpec.Metadata.Name;
+
+            return new RunningService(name, result);
+        }
+
+        private List<ContainerRecipePortMapEntry> ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes)
+        {
+            var result = new List<ContainerRecipePortMapEntry>();
+
+            // For each container-recipe-port, we need to figure out which service-ports it was assigned by K8s.
+            var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sNamespace));
+            foreach (var r in containerRecipes)
+            {
+                var recipePorts = r.ExposedPorts.Concat(r.InternalPorts).ToArray();
+                foreach (var port in recipePorts)
+                {
+                    var portName = GetNameForPort(r, port);
+
+                    var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName);
+                    var ports = matchingServicePorts.Select(p => MapPortIfAble(p, port.Tag, port.Protocol)).ToArray();
+
+                    if (ports.Any())
+                    {
+                        result.Add(new ContainerRecipePortMapEntry(r.Number, ports));
+                        log.Debug($"Service Readback: {portName} found: {string.Join(",", ports.Select(p => p.ToString()))}");
+                    }
+                }
+            }
+
+            return result;
+        }
+
+        private Port MapPortIfAble(V1ServicePort p, string tag, PortProtocol protocol)
+        {
+            if (p.NodePort != null) return new Port(p.NodePort.Value, tag, protocol);
+            if (p.Port > 0) return new Port(p.Port, tag, protocol);
+            throw new Exception("Unable to map port.");
+        }
+
+        private void DeleteService(RunningService service)
+        {
+            client.Run(c => c.DeleteNamespacedService(service.Name, K8sNamespace));
+        }
+
+        private V1ObjectMeta CreateServiceMetadata(ContainerRecipe[] containerRecipes, string namePostfix)
+        {
+            var recipeName = containerRecipes.First().Name;
+            var name = K8sNameUtils.Format($"{recipeName}-{workflowNumberSource.WorkflowNumber}-{namePostfix}");
+            log.Debug("Creating service: " + name);
+            return new V1ObjectMeta
+            {
+                Name = name,
+                NamespaceProperty = K8sNamespace,
+            };
+        }
+
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector, bool isNodePort)
+        {
+            var result = new List<V1ServicePort>();
+            foreach (var recipe in recipes)
+            {
+                var ports = portSelector(recipe);
+                foreach (var port in ports)
+                {
+                    result.AddRange(CreateServicePorts(recipe, port, isNodePort));
+                }
+            }
+            return result;
+        }
+
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe, Port recipePort, bool isNodePort)
+        {
+            var result = new List<V1ServicePort>();
+            if (recipePort.IsTcp()) CreateServicePort(result, recipe, recipePort, "TCP", isNodePort);
+            if (recipePort.IsUdp()) CreateServicePort(result, recipe, recipePort, "UDP", isNodePort);
+            return result;
+        }
+
+        private void CreateServicePort(List<V1ServicePort> result, ContainerRecipe recipe, Port port, string protocol, bool isNodePort)
+        {
+            var p = new V1ServicePort
+            {
+                Name = GetNameForPort(recipe, port),
+                Protocol = protocol,
+                Port = port.Number,
+                TargetPort = GetNameForPort(recipe, port)
+            };
+
+            if (isNodePort) p.NodePort = port.Number;
+
+            result.Add(p);
+        }
+
+        #endregion
+
+        #region Waiting
+
+        private void WaitUntilNamespaceCreated() 
+        {
+            WaitUntil(() => IsNamespaceOnline(K8sNamespace), nameof(WaitUntilNamespaceCreated));
+        }
+
+        private void WaitUntilNamespaceDeleted(string @namespace)
+        {
+            WaitUntil(() => !IsNamespaceOnline(@namespace), nameof(WaitUntilNamespaceDeleted));
+        }
+
+        private void WaitUntilDeploymentOnline(RunningContainer container)
+        {
+            WaitUntil(() =>
+            {
+                CheckForCrash(container);
+
+                var deployment = client.Run(c => c.ReadNamespacedDeployment(container.Recipe.Name, K8sNamespace));
+                return deployment?.Status.AvailableReplicas != null && deployment.Status.AvailableReplicas > 0;
+            }, nameof(WaitUntilDeploymentOnline));
+        }
+
+        private void CheckForCrash(RunningContainer container)
+        {
+            var deploymentName = container.Recipe.Name;
+            var podName = GetPodName(container);
+
+            var podInfo = client.Run(c => c.ReadNamespacedPod(podName, K8sNamespace));
+            if (podInfo == null) return;
+            if (podInfo.Status == null) return;
+            if (podInfo.Status.ContainerStatuses == null) return;
+
+            var result = podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+            if (result)
+            {
+                var msg = $"Pod crash detected for deployment {deploymentName} (pod:{podName})";
+                log.Error(msg);
+
+                DownloadPodLog(container, new WriteToFileLogHandler(log, msg), tailLines: null, previous: true);
+
+                throw new Exception(msg);
+            }
+        }
+
+        private void WaitUntilDeploymentOffline(string deploymentName)
+        {
+            WaitUntil(() =>
+            {
+                var deployments = client.Run(c => c.ListNamespacedDeployment(K8sNamespace));
+                var deployment = deployments.Items.SingleOrDefault(d => d.Metadata.Name == deploymentName);
+                return deployment == null || deployment.Status.AvailableReplicas == 0;
+            }, nameof(WaitUntilDeploymentOffline));
+        }
+
+        private void WaitUntilPodsForDeploymentAreOffline(RunningDeployment deployment)
+        {
+            WaitUntil(() =>
+            {
+                var pods = FindPodsByLabel(deployment.PodLabel);
+                return !pods.Any();
+            }, nameof(WaitUntilPodsForDeploymentAreOffline));
+        }
+
+        private void WaitUntil(Func<bool> predicate, string msg)
+        {
+            var sw = Stopwatch.Begin(log, true);
+            try
+            {
+                Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay(), msg);
+            }
+            finally
+            {
+                sw.End(msg, 1);
+            }
+        }
+
+        #endregion
+
+        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        {
+            var containerName = container.Name;
+            var podName = GetPodName(container);
+            var recipeName = container.Recipe.Name;
+
+            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
+        }
+
+        private V1Pod[] FindPodsByLabel(string podLabel)
+        {
+            var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace));
+            return pods.Items.Where(p => p.GetLabel(PodLabelKey) == podLabel).ToArray();
+        }
+
+        private PodInfo CreatePodInfo(V1Pod pod)
+        {
+            var name = pod.Name();
+            var ip = pod.Status.PodIP;
+            var k8sNodeName = pod.Spec.NodeName;
+
+            if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure.");
+            if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure.");
+
+            return new PodInfo(name, ip, k8sNodeName);
+        }
+    }
+}

Framework/KubernetesWorkflow/K8sHooks.cs

@@ -0,0 +1,27 @@
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
+
+namespace KubernetesWorkflow
+{
+    public interface IK8sHooks
+    {
+        void OnContainersStarted(RunningPod runningPod);
+        void OnContainersStopped(RunningPod runningPod);
+        void OnContainerRecipeCreated(ContainerRecipe recipe);
+    }
+
+    public class DoNothingK8sHooks : IK8sHooks
+    {
+        public void OnContainersStarted(RunningPod runningPod)
+        {
+        }
+
+        public void OnContainersStopped(RunningPod runningPod)
+        {
+        }
+
+        public void OnContainerRecipeCreated(ContainerRecipe recipe)
+        {
+        }
+    }
+}

Framework/KubernetesWorkflow/K8sNameUtils.cs

@@ -0,0 +1,33 @@
+namespace KubernetesWorkflow
+{
+    public static class K8sNameUtils
+    {
+        public static string Format(string s)
+        {
+            return Format(s, 62);
+        }
+
+        public static string FormatPortName(string s)
+        {
+            return Format(s, 15);
+        }
+
+        private static string Format(string s, int maxLength)
+        {
+            var result = s.ToLowerInvariant()
+                .Replace("_", "-")
+                .Replace(" ", "-")
+                .Replace(":", "-")
+                .Replace("/", "-")
+                .Replace("\\", "-")
+                .Replace("[", "-")
+                .Replace("]", "-")
+                .Replace(",", "-");
+
+            result = result.Trim('-');
+            if (result.Length > maxLength) result = result.Substring(0, maxLength);
+
+            return result;
+        }
+    }
+}

Framework/KubernetesWorkflow/KnownLocations.cs

@@ -0,0 +1,47 @@
+namespace KubernetesWorkflow
+{
+    public interface IKnownLocations
+    {
+        /// <summary>
+        /// Returns a known location given an index.
+        /// Each index guarantees a different location.
+        /// </summary>
+        ILocation Get(int index);
+        int NumberOfLocations { get; }
+
+        /// <summary>
+        /// Returns the location object for a specific kubernetes node. Throws if it doesn't exist.
+        /// </summary>
+        ILocation Get(string kubeNodeName, bool allowPartialMatches = false);
+    }
+
+    public class KnownLocations : IKnownLocations
+    {
+        private readonly Location[] locations;
+
+        public KnownLocations(Location[] locations)
+        {
+            this.locations = locations;
+            if (locations.Any(l => l.NodeLabel == null)) throw new Exception("Must not contain unspecified location");
+        }
+
+        public static ILocation UnspecifiedLocation { get; } = new Location();
+
+        public int NumberOfLocations => locations.Length;
+
+        public ILocation Get(int index)
+        {
+            return locations[index];
+        }
+
+        public ILocation Get(string kubeNodeName, bool allowPartialMatches = false)
+        {
+            if (allowPartialMatches)
+            {
+                return locations.Single(l => l.NodeLabel != null && l.NodeLabel.Value.Contains(kubeNodeName));
+            }
+
+            return locations.Single(l => l.NodeLabel != null && l.NodeLabel.Value == kubeNodeName);
+        }
+    }
+}

Framework/KubernetesWorkflow/KubernetesWorkflow.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 	  <RootNamespace>KubernetesWorkflow</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>
@@ -9,9 +9,11 @@
 
   <ItemGroup>
 	  <PackageReference Include="KubernetesClient" Version="10.1.4" />
+	  <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\Logging\Logging.csproj" />
     <ProjectReference Include="..\Utils\Utils.csproj" />
   </ItemGroup>
 

Framework/KubernetesWorkflow/Location.cs

@@ -0,0 +1,24 @@
+using KubernetesWorkflow.Types;
+
+namespace KubernetesWorkflow
+{
+    public interface ILocation
+    {
+    }
+
+    public class Location : ILocation
+    {
+        internal Location(K8sNodeLabel? nodeLabel = null)
+        {
+            NodeLabel = nodeLabel;
+        }
+
+        internal K8sNodeLabel? NodeLabel { get; }
+
+        public override string ToString()
+        {
+            if (NodeLabel == null) return "Location:Unspecified";
+            return $"Location:KubeNode-'{NodeLabel.Key}:{NodeLabel.Value}'";
+        }
+    }
+}

Framework/KubernetesWorkflow/LocationProvider.cs

@@ -0,0 +1,50 @@
+using KubernetesWorkflow.Types;
+using Logging;
+
+namespace KubernetesWorkflow
+{
+    public class LocationProvider
+    {
+        private readonly TimeSpan locationsExpirationTime = TimeSpan.FromMinutes(10);
+        private readonly ILog log;
+        private readonly Action<Action<K8sController>> onController;
+        private Location[] knownLocations = Array.Empty<Location>();
+        private DateTime lastUpdate = DateTime.UtcNow;
+
+        public LocationProvider(ILog log, Action<Action<K8sController>> onController)
+        {
+            this.log = log;
+            this.onController = onController;
+        }
+
+        public IKnownLocations GetAvailableLocations()
+        {
+            if (ShouldUpdateKnownLocations())
+            {
+                onController(UpdateKnownLocations);
+            }
+
+            return new KnownLocations(knownLocations);
+        }
+
+        private void UpdateKnownLocations(K8sController controller)
+        {
+            knownLocations = controller.GetAvailableK8sNodes().Select(CreateLocation).ToArray();
+            lastUpdate = DateTime.UtcNow;
+
+            log.Log($"Detected {knownLocations.Length} available locations: '{string.Join(",", knownLocations.Select(l => l.ToString()))}'");
+        }
+
+        private Location CreateLocation(K8sNodeLabel k8sNode)
+        {
+            return new Location(k8sNode);
+        }
+
+        private bool ShouldUpdateKnownLocations()
+        {
+            if (!knownLocations.Any()) return true;
+            if (DateTime.UtcNow - lastUpdate > locationsExpirationTime) return true;
+            return false;
+        }
+    }
+}

Framework/KubernetesWorkflow/LogHandler.cs

@@ -0,0 +1,46 @@
+using Logging;
+
+namespace KubernetesWorkflow
+{
+    public interface ILogHandler
+    {
+        void Log(Stream log);
+    }
+
+    public abstract class LogHandler : ILogHandler
+    {
+        public void Log(Stream log)
+        {
+            using var reader = new StreamReader(log);
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                ProcessLine(line);
+                line = reader.ReadLine();
+            }
+        }
+
+        protected abstract void ProcessLine(string line);
+    }
+
+    public class WriteToFileLogHandler : LogHandler, ILogHandler
+    {
+        public WriteToFileLogHandler(ILog sourceLog, string description)
+        {
+            LogFile = sourceLog.CreateSubfile();
+
+            var msg = $"{description} -->> {LogFile.FullFilename}";
+            sourceLog.Log(msg);
+
+            LogFile.Write(msg);
+            LogFile.WriteRaw(description);
+        }
+
+        public LogFile LogFile { get; }
+
+        protected override void ProcessLine(string line)
+        {
+            LogFile.WriteRaw(line);
+        }
+    }
+}

Framework/KubernetesWorkflow/PublicIpService.cs

@@ -0,0 +1,7 @@
+namespace KubernetesWorkflow
+{
+    public static class PublicIpService
+    {
+        public static string Address { get; } = "ip.codex.storage";
+    }
+}

Framework/KubernetesWorkflow/Recipe/CommandOverride.cs

@@ -0,0 +1,12 @@
+namespace KubernetesWorkflow.Recipe
+{
+    public class CommandOverride
+    {
+        public CommandOverride(params string[] command)
+        {
+            Command = command;
+        }
+
+        public string[] Command { get; }
+    }
+}

Framework/KubernetesWorkflow/Recipe/ContainerAdditionals.cs

@@ -0,0 +1,53 @@
+using Newtonsoft.Json.Linq;
+
+namespace KubernetesWorkflow.Recipe
+{
+    public class ContainerAdditionals
+    {
+        public ContainerAdditionals(Additional[] additionals)
+        {
+            Additionals = additionals;
+        }
+
+        public static ContainerAdditionals CreateFromUserData(IEnumerable<object> userData)
+        {
+            return new ContainerAdditionals(userData.Select(ConvertToAdditional).ToArray());
+        }
+
+        public Additional[] Additionals { get; }
+
+        public T? Get<T>()
+        {
+            var typeName = GetTypeName(typeof(T));
+            var userData = Additionals.SingleOrDefault(a => a.Type == typeName);
+            if (userData == null) return default;
+            var jobject = (JObject)userData.UserData;
+            return jobject.ToObject<T>();
+        }
+
+        private static Additional ConvertToAdditional(object userData)
+        {
+            var typeName = GetTypeName(userData.GetType());
+            return new Additional(typeName, userData);
+        }
+
+        private static string GetTypeName(Type type)
+        {
+            var typeName = type.FullName;
+            if (string.IsNullOrEmpty(typeName)) throw new Exception("Object type fullname is null or empty: " + type);
+            return typeName;
+        }
+    }
+
+    public class Additional
+    {
+        public Additional(string type, object userData)
+        {
+            Type = type;
+            UserData = userData;
+        }
+
+        public string Type { get; }
+        public object UserData { get; }
+    }
+}

Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs

@@ -0,0 +1,144 @@
+namespace KubernetesWorkflow.Recipe
+{
+    public class ContainerRecipe
+    {
+        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, CommandOverride commandOverride, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
+        {
+            Number = number;
+            NameOverride = nameOverride;
+            Image = image;
+            Resources = resources;
+            SchedulingAffinity = schedulingAffinity;
+            CommandOverride = commandOverride;
+            SetCriticalPriority = setCriticalPriority;
+            ExposedPorts = exposedPorts;
+            InternalPorts = internalPorts;
+            EnvVars = envVars;
+            PodLabels = podLabels;
+            PodAnnotations = podAnnotations;
+            Volumes = volumes;
+            Additionals = additionals;
+
+            if (NameOverride != null)
+            {
+                Name = $"{K8sNameUtils.Format(NameOverride)}-{Number}";
+            }
+            else
+            {
+                Name = $"ctnr{Number}";
+            }
+
+            if (exposedPorts.Any(p => string.IsNullOrEmpty(p.Tag))) throw new Exception("Port tags are required for all exposed ports.");
+        }
+
+        public string Name { get; }
+        public int Number { get; }
+        public string? NameOverride { get; }
+        public ContainerResources Resources { get; }
+        public SchedulingAffinity SchedulingAffinity { get; }
+        public CommandOverride CommandOverride { get; }
+        public bool SetCriticalPriority { get; }
+        public string Image { get; }
+        public Port[] ExposedPorts { get; }
+        public Port[] InternalPorts { get; }
+        public EnvVar[] EnvVars { get; }
+        public PodLabels PodLabels { get; }
+        public PodAnnotations PodAnnotations { get; }
+        public VolumeMount[] Volumes { get; }
+        public ContainerAdditionals Additionals { get; }
+
+        public Port? GetPortByTag(string tag)
+        {
+            return ExposedPorts.Concat(InternalPorts).SingleOrDefault(p => p.Tag == tag);
+        }
+
+        public override string ToString()
+        {
+            return $"(container-recipe: {Name}, image: {Image}, " +
+                $"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " +
+                $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
+                $"envVars: {string.Join(",", EnvVars.Select(v => v.ToString()))}, " +
+                $"limits: {Resources}, " +
+                $"affinity: {SchedulingAffinity}, " +
+                $"volumes: {string.Join(",", Volumes.Select(v => $"'{v.MountPath}'"))}";
+        }
+    }
+
+    public class Port
+    {
+        public Port(int number, string tag, PortProtocol protocol)
+        {
+            Number = number;
+            Tag = tag;
+            Protocol = protocol;
+
+            if (string.IsNullOrWhiteSpace(Tag))
+            {
+                throw new Exception("A unique port tag is required");
+            }
+        }
+
+        public int Number { get; }
+        public string Tag { get; }
+        public PortProtocol Protocol { get; }
+
+        public bool IsTcp()
+        {
+            return Protocol == PortProtocol.TCP;
+        }
+
+        public bool IsUdp()
+        {
+            return Protocol == PortProtocol.UDP;
+        }
+
+        public override string ToString()
+        {
+            if (string.IsNullOrEmpty(Tag)) return $"untagged-port={Number}/{Protocol}";
+            return $"{Tag}={Number}/{Protocol}";
+        }
+    }
+
+    public enum PortProtocol
+    {
+        TCP,
+        UDP
+    }
+
+    public class EnvVar
+    {
+        public EnvVar(string name, string value)
+        {
+            Name = name;
+            Value = value;
+        }
+
+        public string Name { get; }
+        public string Value { get; }
+
+        public override string ToString()
+        {
+            return $"'{Name}' = '{Value}'";
+        }
+    }
+
+    public class VolumeMount
+    {
+        public VolumeMount(string volumeName, string mountPath, string? subPath = null, string? resourceQuantity = null, string? secret = null, string? hostPath = null)
+        {
+            VolumeName = volumeName;
+            MountPath = mountPath;
+            SubPath = subPath;
+            ResourceQuantity = resourceQuantity;
+            Secret = secret;
+            HostPath = hostPath;
+        }
+
+        public string VolumeName { get; }
+        public string MountPath { get; }
+        public string? SubPath { get; }
+        public string? ResourceQuantity { get; }
+        public string? Secret { get; }
+        public string? HostPath { get; }
+    }
+}

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -0,0 +1,168 @@
+using Utils;
+
+namespace KubernetesWorkflow.Recipe
+{
+    public abstract class ContainerRecipeFactory
+    {
+        private readonly List<Port> exposedPorts = new List<Port>();
+        private readonly List<Port> internalPorts = new List<Port>();
+        private readonly List<EnvVar> envVars = new List<EnvVar>();
+        private readonly PodLabels podLabels = new PodLabels();
+        private readonly PodAnnotations podAnnotations = new PodAnnotations();
+        private readonly List<VolumeMount> volumeMounts = new List<VolumeMount>();
+        private readonly List<object> additionals = new List<object>();
+        private RecipeComponentFactory factory = null!;
+        private ContainerResources resources = new ContainerResources();
+        private SchedulingAffinity schedulingAffinity = new SchedulingAffinity();
+        private CommandOverride commandOverride = new CommandOverride();
+        private bool setCriticalPriority;
+
+        public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config)
+        {
+            this.factory = factory;
+            ContainerNumber = containerNumber;
+            Index = index;
+
+            Initialize(config);
+
+            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, commandOverride, setCriticalPriority,
+                exposedPorts.ToArray(),
+                internalPorts.ToArray(),
+                envVars.ToArray(),
+                podLabels.Clone(),
+                podAnnotations.Clone(),
+                volumeMounts.ToArray(),
+                ContainerAdditionals.CreateFromUserData(additionals));
+
+            exposedPorts.Clear();
+            internalPorts.Clear();
+            envVars.Clear();
+            podLabels.Clear();
+            podAnnotations.Clear();
+            volumeMounts.Clear();
+            additionals.Clear();
+            this.factory = null!;
+            resources = new ContainerResources();
+            schedulingAffinity = new SchedulingAffinity();
+            commandOverride = new CommandOverride();
+            setCriticalPriority = false;
+
+            return recipe;
+        }
+
+        public abstract string AppName { get; }
+        public abstract string Image { get; }
+        protected int ContainerNumber { get; private set; } = 0;
+        protected int Index { get; private set; } = 0;
+        protected abstract void Initialize(StartupConfig config);
+
+        protected Port AddExposedPort(string tag, PortProtocol protocol = PortProtocol.TCP)
+        {
+            return AddExposedPort(factory.CreateExternalPort(tag, protocol));
+        }
+
+        protected Port AddExposedPort(int number, string tag, PortProtocol protocol = PortProtocol.TCP)
+        {
+            return AddExposedPort(factory.CreateExternalPort(number, tag, protocol));
+        }
+
+        protected Port AddInternalPort(string tag = "", PortProtocol protocol = PortProtocol.TCP)
+        {
+            var p = factory.CreateInternalPort(tag, protocol);
+            internalPorts.Add(p);
+            return p;
+        }
+
+        protected void AddExposedPortAndVar(string name, string tag, PortProtocol protocol = PortProtocol.TCP)
+        {
+            AddEnvVar(name, AddExposedPort(tag, protocol));
+        }
+
+        protected void AddInternalPortAndVar(string name, string tag = "", PortProtocol protocol = PortProtocol.TCP)
+        {
+            AddEnvVar(name, AddInternalPort(tag, protocol));
+        }
+
+        protected void AddEnvVar(string name, string value)
+        {
+            envVars.Add(factory.CreateEnvVar(name, value));
+        }
+
+        protected void AddEnvVar(string name, Port value)
+        {
+            envVars.Add(factory.CreateEnvVar(name, value.Number));
+        }
+
+        protected void AddPodLabel(string name, string value)
+        {
+            podLabels.Add(name, value);
+        }
+
+        protected void AddPodAnnotation(string name, string value)
+        {
+            podAnnotations.Add(name, value);
+        }
+
+        protected void AddVolume(string name, string mountPath, string? subPath = null, string? secret = null, string? hostPath = null)
+        {
+            var size = 10.MB().SizeInBytes.ToString();
+            volumeMounts.Add(new VolumeMount(name, mountPath, subPath, size, secret, hostPath));
+        }
+
+        protected void AddVolume(string mountPath, ByteSize volumeSize)
+        {
+            volumeMounts.Add(new VolumeMount(
+                $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}",
+                mountPath,
+                resourceQuantity: volumeSize.SizeInBytes.ToString()));
+        }
+
+        protected void Additional(object userData)
+        {
+            additionals.Add(userData);
+        }
+
+        protected void SetResourcesRequest(int milliCPUs, ByteSize memory)
+        {
+            SetResourcesRequest(new ContainerResourceSet(milliCPUs, memory));
+        }
+
+        protected void SetSchedulingAffinity(string notIn)
+        {
+            schedulingAffinity = new SchedulingAffinity(notIn);
+        }
+
+        protected void OverrideCommand(params string[] command)
+        {
+            commandOverride = new CommandOverride(command);
+        }
+
+        protected void SetSystemCriticalPriority()
+        {
+            setCriticalPriority = true;
+        }
+
+        // Disabled following a possible bug in the k8s cluster that will throttle containers much more than is
+        // called for if they have resource limits defined.
+        //protected void SetResourceLimits(int milliCPUs, ByteSize memory)
+        //{
+        //    SetResourceLimits(new ContainerResourceSet(milliCPUs, memory));
+        //}
+
+        protected void SetResourcesRequest(ContainerResourceSet requests)
+        {
+            resources.Requests = requests;
+        }
+
+        protected void SetResourceLimits(ContainerResourceSet limits)
+        {
+            resources.Limits = limits;
+        }
+
+        private Port AddExposedPort(Port port)
+        {
+            exposedPorts.Add(port);
+            return port;
+        }
+    }
+}

Framework/KubernetesWorkflow/Recipe/ContainerResources.cs

@@ -0,0 +1,52 @@
+using Utils;
+
+namespace KubernetesWorkflow.Recipe
+{
+    public class ContainerResources
+    {
+        public ContainerResourceSet Requests { get; set; } = new ContainerResourceSet();
+        public ContainerResourceSet Limits { get; set; } = new ContainerResourceSet();
+
+        public override string ToString()
+        {
+            return $"requests:{Requests}, limits:{Limits}";
+        }
+    }
+
+    public class ContainerResourceSet
+    {
+        public ContainerResourceSet(int milliCPUs, ByteSize memory)
+        {
+            MilliCPUs = milliCPUs;
+            Memory = memory;
+        }
+
+        public ContainerResourceSet(int milliCPUs)
+            : this(milliCPUs, new ByteSize(0))
+        {
+        }
+
+        public ContainerResourceSet(ByteSize memory)
+            : this(0, memory)
+        {
+        }
+
+        public ContainerResourceSet()
+            : this(0)
+        {
+        }
+
+        public int MilliCPUs { get; }
+        public ByteSize Memory { get; }
+
+        public override string ToString()
+        {
+            var result = new List<string>();
+            if (MilliCPUs == 0) result.Add("cpu: unlimited");
+            else result.Add($"cpu: {MilliCPUs} milliCPUs");
+            if (Memory.SizeInBytes == 0) result.Add("memory: unlimited");
+            else result.Add($"memory: {Memory}");
+            return string.Join(", ", result);
+        }
+    }
+}

Framework/KubernetesWorkflow/Recipe/PodAnnotations.cs

@@ -0,0 +1,29 @@
+namespace KubernetesWorkflow.Recipe
+{
+    public class PodAnnotations
+    {
+        private readonly Dictionary<string, string> annotations = new Dictionary<string, string>();
+
+        public void Add(string key, string value)
+        {
+            annotations.Add(key, value);
+        }
+
+        public PodAnnotations Clone()
+        {
+            var result = new PodAnnotations();
+            foreach (var entry in annotations) result.Add(entry.Key, entry.Value);
+            return result;
+        }
+
+        public void Clear()
+        {
+            annotations.Clear();
+        }
+
+        internal Dictionary<string, string> GetAnnotations()
+        {
+            return annotations;
+        }
+    }
+}

Framework/KubernetesWorkflow/Recipe/PodLabels.cs

@@ -0,0 +1,29 @@
+namespace KubernetesWorkflow.Recipe
+{
+    public class PodLabels
+    {
+        private readonly Dictionary<string, string> labels = new Dictionary<string, string>();
+
+        public void Add(string key, string value)
+        {
+            labels.Add(key, K8sNameUtils.Format(value));
+        }
+
+        public PodLabels Clone()
+        {
+            var result = new PodLabels();
+            foreach (var entry in labels) result.Add(entry.Key, entry.Value);
+            return result;
+        }
+
+        public void Clear()
+        {
+            labels.Clear();
+        }
+
+        internal Dictionary<string, string> GetLabels()
+        {
+            return labels;
+        }
+    }
+}

Framework/KubernetesWorkflow/Recipe/RecipeComponentFactory.cs

@@ -0,0 +1,50 @@
+using System.Globalization;
+using Utils;
+
+namespace KubernetesWorkflow.Recipe
+{
+    public class RecipeComponentFactory
+    {
+        private readonly NumberSource internalNumberSource = new NumberSource(8080);
+        private static readonly NumberSource externalNumberSource = new NumberSource(30000);
+        private static int[] usedExternalPorts = Array.Empty<int>();
+
+        public void Update(K8sController controller)
+        {
+            usedExternalPorts = controller.GetUsedExternalPorts();
+        }
+
+        public Port CreateInternalPort(string tag, PortProtocol protocol)
+        {
+            return new Port(internalNumberSource.GetNextNumber(), tag, protocol);
+        }
+
+        public Port CreateExternalPort(int number, string tag, PortProtocol protocol)
+        {
+            if (usedExternalPorts.Contains(number)) throw new Exception($"External port number {number} is already in use by the cluster.");
+            return new Port(number, tag, protocol);
+        }
+
+        public Port CreateExternalPort(string tag, PortProtocol protocol)
+        {
+            while (true)
+            {
+                var number = externalNumberSource.GetNextNumber();
+                if (!usedExternalPorts.Contains(number))
+                {
+                    return new Port(number, tag, protocol);
+                }
+            }
+        }
+
+        public EnvVar CreateEnvVar(string name, int value)
+        {
+            return CreateEnvVar(name, value.ToString(CultureInfo.InvariantCulture));
+        }
+
+        public EnvVar CreateEnvVar(string name, string value)
+        {
+            return new EnvVar(name, value);
+        }
+    }
+}