Overdue readme updates

# Conflicts:
#	ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

.github/workflows/dist-tests.yaml

@@ -1,81 +0,0 @@
-name: Dist Tests
-
-
-on:
-  # push:
-  #   branches:
-  #     - master
-  #   tags:
-  #     - 'v*.*.*'
-  #   paths-ignore:
-  #     - '**/*.md'
-  #     - '.gitignore'
-  #     - 'docker/**'
-  #     - '!docker/job.yaml'
-  workflow_dispatch:
-    inputs:
-      branch:
-        description: Branch (master)
-        required: false
-        type: string
-      source:
-        description: Repository with tests (current)
-        required: false
-        type: string
-      nameprefix:
-        description: Runner prefix (codex-dist-tests)
-        required: false
-        type: string
-      namespace:
-        description: Runner namespace (default)
-        required: false
-        type: string
-      command:
-        description: Runner command (dotnet test Tests)
-        required: false
-        type: string
-
-
-env:
-  BRANCH: ${{ github.ref_name }}
-  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
-  NAMEPREFIX: codex-dist-tests
-  NAMESPACE: default
-  COMMAND: dotnet test Tests/CodexTests
-  JOB_MANIFEST: docker/job.yaml
-  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
-  KUBE_VERSION: v1.26.1
-
-
-jobs:
-  run_tests:
-    name: Run Tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Variables
-        run: |
-          [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.command }}" ]] && COMMAND="${{ github.event.inputs.command }}" || COMMAND="${{ env.COMMAND }}"
-          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'$COMMAND'"')" >>"$GITHUB_ENV"
-          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
-          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
-
-      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
-        uses: azure/setup-kubectl@v3
-        with:
-          version: ${{ env.KUBE_VERSION }}
-
-      - name: Kubectl - Kubeconfig
-        run: |
-          mkdir -p "${HOME}"/.kube
-          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
-
-      - name: Kubectl - Create Job
-        run: |
-          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -

.github/workflows/docker-discordbot.yml

@@ -0,0 +1,28 @@
+name: Docker - Discord Bot
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/BiblioTech/**'
+      - '!Tools/BiblioTech/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-discordbot.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/BiblioTech/docker/Dockerfile
+      docker_repo: codexstorage/codex-discordbot
+    secrets: inherit
+

.github/workflows/docker-reusable.yml

@@ -1,4 +1,4 @@
-name: Docker - Reusable
+name: Reusable - Docker
 
 
 on:
@@ -66,26 +66,26 @@ jobs:
       PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Docker - Meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.DOCKER_REPO }}
 
       - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Docker - Build and Push by digest
         id: build
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ${{ env.DOCKER_FILE }}
@@ -101,10 +101,10 @@ jobs:
           touch "/tmp/digests/${digest#sha256:}"
 
       - name: Docker - Upload digest
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: digests
-          path: /tmp/digests/*
+          name: digests-${{ matrix.target.arch }}
+          path: /tmp/digests
           if-no-files-found: error
           retention-days: 1
 
@@ -139,17 +139,18 @@ jobs:
           fi
 
       - name: Docker - Download digests
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: digests
+          pattern: digests-*
+          merge-multiple: true
           path: /tmp/digests
 
       - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Docker - Meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.DOCKER_REPO }}
           flavor: |
@@ -161,7 +162,7 @@ jobs:
             type=sha,enable=${{ env.TAG_SHA }}
 
       - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/docker-rewarder.yml

@@ -0,0 +1,26 @@
+name: Docker - Rewarder Bot
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/TestNetRewarder/**'
+      - '!Tools/TestNetRewarder/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-rewarder.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/TestNetRewarder/docker/Dockerfile
+      docker_repo: codexstorage/codex-rewarderbot
+    secrets: inherit
+

.github/workflows/docker-runner.yml

@@ -1,4 +1,4 @@
-name: Docker
+name: Docker - Runner
 
 
 on:
@@ -10,7 +10,7 @@ on:
     paths:
       - docker/Dockerfile
       - docker/docker-entrypoint.sh
-      - .github/workflows/docker.yml
+      - .github/workflows/docker-runner.yml
       - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 

.github/workflows/report-HoldMyBeerTest.yaml

@@ -0,0 +1,20 @@
+name: Report - HoldMyBeerTest
+
+
+on:
+  # schedule:
+  #   - cron: '30 1 */2 * *'
+  workflow_dispatch:
+
+jobs:
+  run_tests:
+    name: Call runner
+    uses: ./.github/workflows/run-continuous-tests.yaml
+    with:
+      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+      branch: master
+      nameprefix: c-tests-report-holdmybeertest-48h
+      tests_target_duration: 48h
+      tests_filter: HoldMyBeerTest
+      tests_cleanup: true
+    secrets: inherit

.github/workflows/report-PeersTest.yaml

@@ -0,0 +1,20 @@
+name: Report - PeersTest
+
+
+on:
+  # schedule:
+  #   - cron: '30 1 */2 * *'
+  workflow_dispatch:
+
+jobs:
+  run_tests:
+    name: Call runner
+    uses: ./.github/workflows/run-continuous-tests.yaml
+    with:
+      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+      branch: master
+      nameprefix: c-tests-report-peerstest-48h
+      tests_target_duration: 48h
+      tests_filter: PeersTest
+      tests_cleanup: true
+    secrets: inherit

.github/workflows/run-continuous-tests.yaml

@@ -0,0 +1,170 @@
+name: Run Continuous Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
+        required: false
+        type: string
+      nameprefix:
+        description: Resources prefix (c-tests)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (2d)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup
+        required: false
+        type: boolean
+        default: true
+  workflow_call:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
+        required: false
+        type: string
+      nameprefix:
+        description: Resources prefix (c-tests)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (2d)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup
+        required: false
+        type: boolean
+        default: true
+      workflow_source:
+        description: Workflow source
+        required: false
+        type: string
+        default: ''
+
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
+  NAMEPREFIX: c-tests
+  NAMESPACE: default
+  TESTS_TARGET_DURATION: 2d
+  TESTS_FILTER: ""
+  TESTS_CLEANUP: true
+  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.28.2
+
+
+jobs:
+  run_tests:
+    name: Run Continuous Tests ${{ inputs.tests_filter }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
+          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          echo "DEPLOYMENT_NAMESPACE=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}"
+          echo "Runner pod: `kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}'`"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+          echo "Tests runid: ${{ env.RUNID }}"
+          echo "Tests codexdockerimage: ${{ env.CODEXDOCKERIMAGE }}"
+          echo "Tests namespace: ${{ env.DEPLOYMENT_NAMESPACE }}"
+          echo "Tests duration: ${{ env.TESTS_TARGET_DURATION }}"
+          echo "Tests filter: ${{ env.TESTS_FILTER }}"
+          echo "Tests cleanup: ${{ env.TESTS_CLEANUP }}"
+          echo "----"
+
+      - name: Show Runner logs
+        if: false
+        run: |
+          # Variables
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          duration=600
+          namespace="${{ env.NAMESPACE }}"
+          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")
+
+          # Check Pod status
+          WAIT=120
+          SECONDS=0
+          sleep=1
+          while (( SECONDS < WAIT )); do
+            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath="{.status.phase}")
+            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in $phase state - Get the logs"; break; } || { echo "Pod $pod is in $phase state - Retry in $sleep seconds / $((WAIT - SECONDS))"; }
+            sleep $sleep
+          done
+
+          # Get logs
+          timeout $duration \
+            kubectl logs $pod \
+              -n $namespace \
+              -f \
+              --tail=-1 \
+              --timestamps || true

.github/workflows/run-dist-tests.yaml

@@ -0,0 +1,83 @@
+name: Run Dist Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      nameprefix:
+        description: Runner name prefix (d-tests-runner)
+        required: false
+        type: string
+      namespace:
+        description: Runner namespace (default)
+        required: false
+        type: string
+      command:
+        description: Command (dotnet test Tests/CodexTests)
+        required: false
+        type: string
+
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  NAMEPREFIX: d-tests-runner
+  NAMESPACE: default
+  COMMAND: dotnet test Tests/CodexTests
+  JOB_MANIFEST: docker/dist-tests-job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.28.2
+
+
+jobs:
+  run_tests:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Variables
+        run: |
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'$COMMAND'"')" >>"$GITHUB_ENV"
+          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v4
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+          echo "Tests runid: ${{ env.RUNID }}"
+          echo "Tests command: `jq -r '. | join(" ")' <<< '${{ env.COMMAND }}'`"
+          echo "----"

CONTRIBUTINGPLUGINS.MD

@@ -1,10 +1,26 @@
 # Distributed System Tests for Nim-Codex
 
 ## Contributing plugins
-The testing framework was created for testing Codex. However, it's been designed such that other distributed/containerized projects can 'easily' be added. In order to add your project to the framework you must:
+The testing framework was created for testing Codex. However, it's been designed such that other containerized projects can 'easily' be added.
+
+In this file, you'll see 'users' (in quote) mentioned once or twice. This refers to code/projects/tests which end up making use of your plugin. 'Users' come in many shapes and sizes and tend to have many differen use-cases in mind. Please consider this when reading this document and writing your plugin.
+
+## Checklist
+Your application must pass this checklist to be compatible with the framework:
+- It runs in a docker container.
+- It can be configured via environment variables. (You may need to create a docker image which contains a shell script, to pass some env-vars as CLI arguments to your application. Container command overrides do work, but are not equally reliable across container platforms. When in doubt: use env-var!)
+- It has network interaction:
+   - It exposes one or more APIs via one or more ports, OR
+   - It makes calls to other services. (OR both.)
+
+If your application's use-cases rely primarily on shell interaction, this framework might not be for you. The framework allows you to execute commands in containers AND read stdout/stderr responses. However, its focus during development has always been webservice API interactions.
+
+## Steps
+In order to add your project to the framework you must:
 1. Create a library assembly in the project plugins folder.
 1. It must contain a type that implements the `IProjectPlugin` interface from the `Core` assembly.
-1. If your plugin wants to expose any specific methods or objects to the code using the framework (the tests and tools), it must implement extensions for the `CoreInterface` type.
+1. If your plugin wants to expose any specific methods or objects to 'users', it must implement extensions for the `CoreInterface` type.
+1. If your plugin wants to run containers of its own project, it must provide a recipe.
 
 ## Constructors & Tools
 Your implementation of `IProjectPlugin` must have a public constructor with a single argument of type `IPluginTools`, for example:
@@ -20,19 +36,34 @@ Your implementation of `IProjectPlugin` must have a public constructor with a si
    }
 ```
 
-`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients for containers. (Without having to figure out addresses manually.)
-
 ## Plugin Interfaces
 The `IProjectPlugin` interface requires the implementation of two methods.
-1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version information at this time if applicable.
-1. `Decommission` - Should your plugin have any active system resources, free them in this method.
+1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version and/or configuration information at this time if applicable.
+1. `Decommission` - Should your plugin have any active system resources, free them in this method. Please note that resources managed by the framework (such as running containers and tracked data files) do *not* need to be manually disposed in this method. `Decommission` is to be used for resources not managed by the framework.
 
 There are a few optional interfaces your plugin may choose to implement. The framework will automatically use these interfaces.
-1. `IHasLogPrefix` - Implementing this interface allows you to provide a string with will be prepended to all log statements made by your plugin.
-1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by code that uses your plugin.
+1. `IHasLogPrefix` - Implementing this interface allows you to provide a string which will be prepended to all log statements made by your plugin. A polite thing to do.
+1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by 'users' of your plugin. Often this data finds its way into log files and container-descriptors in order to help track versions/tests/deployments, etc.
+
+## IPluginTools
+`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients to make calls to containers. (Figure out addresses and ports for containers is handled by the framework.)
+
+It is possible and allowed for your plugin to depend on and use other plugins. (For example, maybe your project wants to interact with Ethereum and wants to use the GethPlugin to talk to a Geth node.) `IPluginTools` is *not* what is used for accessing functionality of other plugins. See 'Core Interface' section.
+
+ILog GetLog();
+IHttp CreateHttp(Action<HttpClient> onClientCreated);
+IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
+IHttp CreateHttp();
+IFileManager GetFileManager();
+
+The plugin tools provide:
+1. `Workflow` - This tool allows you to start and stop containers using "container recipes". (More on those below.) It also allows you to execute commands inside a container, access stdout/stderr, detect crashes, and access pod deployment information. The workflow tool also lets you inspect the locations available in the cluster, and decide where you want to run containers. (More on that below as well.)
+1. `Log` - Good logging is priceless. Use this tool to get a log object handle, and write useful debug/info/error statements.
+1. `Http` - This tool gives you a convenient way to access a standard dotnet HttpClient, and takes care of timeouts and retries (in accordance with the config). Additionally, it combos nicely with container objects created by `Workflow`, such that you never have to spend any time figuring out the addresses and ports of your containers.
+1. `FileManager` - Lets you use tracked temporary files. Even if the 'user' tests/application start crashing, the framework will make sure these are cleaned up.
 
 ## Core Interface
-Any functionality your plugin wants to expose to code which uses the framework will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type:
+Any functionality your plugin wants to expose to 'users' will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type:
 ```C#
    public static class CoreInterfaceExtensions
    {
@@ -48,12 +79,14 @@ Any functionality your plugin wants to expose to code which uses the framework w
    }
 ```
 
+If your plugin wants to access the functionality exposed by other plugins, then you can pass the argument `CoreInterface ci` to your plugin code in order to do so. (For example, if you want to start a Geth node, the Geth plugin adds `IGethNode StartGethNode(this CoreInterface ci, Action<IGethSetup> setup)` to the core interface.) Don't forget you'll need to add a project reference to each plugin project you wish to use.
+
 While technically you can build whatever you like on top of the `CoreInterface` and your own plugin types, I recommend that you follow the approach explained below.
 
 ## Deploying, Wrapping, and Starting
 When building a plugin, it is important to make as few assumptions as possible about how it will be used by whoever is going to use the framework. For this reason, I recommend you expose three kinds of methods using your `CoreInterface` extensions:
-1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning containers as a result. If your project requires additional information, you can create a new class type to contain both it and the containers created.
-1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain.
+1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning container objects as a result. If your project requires additional information, you can create a new class type to contain both it and the container objects created.
+1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain. (For example, if my project has a REST API call that allows users to fetch some state information, the object returned by Wrap should have a convenient method to call that API and receive that state information.)
 1. Start - This kind of method does both, simply calling a Deploy method first, then a Wrap method, and returns the result.
 
 Here's an example:
@@ -69,8 +102,8 @@ public static class CoreInterfaceExtensions
 
       public static IMyProjectNode WrapMyProjectContainer(this CoreInterface ci, RunningContainers container)
       {
-         return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` tool to create an HTTP client for the container, then wrap it in an object that
-         // represents the API of your project.
+         return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` to create an HTTP client for the container, then wrap it in an object that
+         // represents the API of your project, in this case 'IMyProjectNode'.
       }
 
       public static IMyProjectNode StartMyProject(this CoreInterface ci, string someArgument)
@@ -82,5 +115,44 @@ public static class CoreInterfaceExtensions
    }
 ```
 
-The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing.
+Should your deploy methods not return framework-types like RunningContainers, please make sure that your custom times are serializable. (Decorate them with the `[Serializable]` attribute.) Tools have been built using this framework which rely on the ability to serialize and store deployment information for later use. Please don't break this possibility. (Consider using the `SerializeGate` type to help ensure compatibility.)
+
+The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinuousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing.
+
+## Container Recipes
+In order to run a container of your application, the framework needs to know how to create that container. Think of a container recipe as being similar to a docker-compose.yaml file: You specify the docker image, ports, environment variables, persistent volumes, and secrets. However, container recipes are code. This allows you to add conditional behaviour to how your container is constructed. For example: The 'user' of your plugin specifies in their call input that they want to run your application in a certain mode. This would cause your container recipe to set certain environment variables, which cause the application to behave in the requested way.
+
+### Addresses and ports
+In a docker-compose.yaml file, it is perfectly normal to specify which ports should be exposed on your container. However, in the framework there's more to consider. When your application container starts, who knows on what kind of machine it runs, and what other processes it's sharing space with? Well, Kubernetes knows. Therefore, it is recommended that container recipes *do not* specify exact port numbers. The framework allows container recipes to declare "a port" without specifying its port number. This allows the framework and Kubernetes to figure out which ports are available when it's time to deploy. In order to find out which port numbers were assigned post-deployment, you can look up the port by tag (which is just an identifying string). When you specify a port to be mapped in your container recipe, you must specify:
+1. `Tag` - An identifier.
+1. `Internal` or `External` - Whether this port should be accessible only inside the cluster (for other containers (k8s: "ClusterIP")) or outside the cluster as well (for external tools/applications (k8s: "NodePort")).
+1. `Protocol` - TCP or UDP. Both protocols on the same port is not universally supported by all container engines, and is therefore not supported by the framework.
+
+If your application wants to listen for incoming traffic from inside its container, be sure to bind it to address "0.0.0.0".
+
+Reminder: If you don't want to worry about addresses, and internal or external ports, you don't have to! The container objects returned by the `workflow` plugin tool have a method called `GetAddress`. Given a port tag, it returns and address object. The `Http` plugin tool can use that address object to set up connections.
+
+## Locations
+The framework is designed to allow you to control instances of your application in multiple (physical) locations. It accomplishes this by using kubernetes, and the ability to deploy containers to specific hosts (nodes) inside a kubernetes cluster. Since Kubernetes allows you to build clusters cross-site, this framework in theory enables you to deploy and interact with containers running anywhere.
+
+The `workflow` plugin tool provides you a list of all available locations in the cluster. When starting a container, you are able to pick one of those locations. If no location is selected, one will be chosen by kubernetes. Locations can be chosen explicitly by kubernetes node name, or, they can be picked from the array of available locations.
+
+Example:
+```C#
+{
+   var location = Ci.GetKnownLocations().Get("kbnode_euwest_paris1");
+   var codex = Ci.StartCodexNode(s => s.At(location));
+}
+```
+In this example, 'Ci' is an instance of the core interface. The CodexPlugin exposes a function 'StartCodexNode', which allows its user to specify a location. This location is then passed to the `workflow` tool when the Codex plugin starts its container.
+
+The available locations array guarantees that each entry corresponds to a different kubernetes host.
+```C#
+{
+   var knownLocations = Ci.GetKnownLocations();
+   // I don't care where exactly, as long as they are different locations.
+   var codexAtZero = Ci.StartCodexNode(s => s.At(knownLocations.Get(0)));
+   var codexAtOne = Ci.StartCodexNode(s => s.At(knownLocations.Get(1)));
+}
+```
 

CONTRIBUTINGTESTS.MD

@@ -21,7 +21,7 @@ Do you want to write some tests using this distributed test setup? Great! Here's
    1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `CodexDistTest` instead.
 1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
 ### Continuous tests
-1. Add new code files to `Tests/CodexContinousTests/Tests`
+1. Add new code files to `Tests/CodexContinuousTests/Tests`
 1. Inherrit from `ContinuousTest`
 1. Define one or more methods and decorate them with the `[TestMoment(...)]` attribute.
 1. The TestMoment takes a number of seconds as argument. Each moment will be executed by the continuous test runner applying the given seconds as delay. (Non-cumulative. So two moments at T:10 will be executed one after another without delay, in this case the order of execution should not be depended upon.)

Framework/Core/CoreInterface.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace Core
 {

Framework/Core/DownloadedLog.cs

@@ -4,7 +4,7 @@ namespace Core
 {
     public interface IDownloadedLog
     {
-        bool DoesLogContain(string expectedString);
+        string[] GetLinesContaining(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
         void DeleteFile();
     }
@@ -18,20 +18,23 @@ namespace Core
             this.logFile = logFile;
         }
 
-        public bool DoesLogContain(string expectedString)
+        public string[] GetLinesContaining(string expectedString)
         {
             using var file = File.OpenRead(logFile.FullFilename);
             using var streamReader = new StreamReader(file);
+            var lines = new List<string>();
 
             var line = streamReader.ReadLine();
             while (line != null)
             {
-                if (line.Contains(expectedString)) return true;
+                if (line.Contains(expectedString))
+                {
+                    lines.Add(line);
+                }
                 line = streamReader.ReadLine();
             }
 
-            //Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
-            return false;
+            return lines.ToArray(); ;
         }
 
         public string[] FindLinesThatContain(params string[] tags)

Framework/Core/Endpoint.cs

@@ -0,0 +1,193 @@
+using Logging;
+using Newtonsoft.Json;
+using Serialization = Newtonsoft.Json.Serialization;
+using System.Net.Http.Headers;
+using System.Net.Http.Json;
+using Utils;
+
+namespace Core
+{
+    public interface IEndpoint
+    {
+        string HttpGetString(string route);
+        T HttpGetJson<T>(string route);
+        TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
+        string HttpPostJson<TRequest>(string route, TRequest body);
+        TResponse HttpPostString<TResponse>(string route, string body);
+        string HttpPostStream(string route, Stream stream);
+        Stream HttpGetStream(string route);
+        T Deserialize<T>(string json);
+    }
+
+    internal class Endpoint : IEndpoint
+    {
+        private readonly ILog log;
+        private readonly IHttp http;
+        private readonly Address address;
+        private readonly string baseUrl;
+        private readonly string? logAlias;
+
+        public Endpoint(ILog log, IHttp http, Address address, string baseUrl, string? logAlias)
+        {
+            this.log = log;
+            this.http = http;
+            this.address = address;
+            this.baseUrl = baseUrl;
+            this.logAlias = logAlias;
+        }
+
+        public string HttpGetString(string route)
+        {
+            return http.OnClient(client =>
+            {
+                return GetString(client, route);
+            }, $"HTTP-GET:{route}");
+        }
+
+        public T HttpGetJson<T>(string route)
+        {
+            return http.OnClient(client =>
+            {
+                var json = GetString(client, route);
+                return Deserialize<T>(json);
+            }, $"HTTP-GET:{route}");
+        }
+
+        public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJson(client, route, body);
+                var json = Time.Wait(response.Content.ReadAsStringAsync());
+                if (!response.IsSuccessStatusCode)
+                {
+                    throw new HttpRequestException(json);
+                }
+                Log(GetUrl() + route, json);
+                return Deserialize<TResponse>(json);
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        public string HttpPostJson<TRequest>(string route, TRequest body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJson(client, route, body);
+                return Time.Wait(response.Content.ReadAsStringAsync());
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        public TResponse HttpPostString<TResponse>(string route, string body)
+        {
+            return http.OnClient(client =>
+            {
+                var response = PostJsonString(client, route, body);
+                if (response == null) throw new Exception("Received no response.");
+                var result = Deserialize<TResponse>(response);
+                if (result == null) throw new Exception("Failed to deserialize response");
+                return result;
+            }, $"HTTO-POST-JSON: {route}");
+        }
+
+        public string HttpPostStream(string route, Stream stream)
+        {
+            return http.OnClient(client =>
+            {
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                var content = new StreamContent(stream);
+                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
+                var response = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(response.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STREAM: {route}");
+        }
+
+        public Stream HttpGetStream(string route)
+        {
+            return http.OnClient(client =>
+            {
+                var url = GetUrl() + route;
+                Log(url, "~ STREAM ~");
+                return Time.Wait(client.GetStreamAsync(url));
+            }, $"HTTP-GET-STREAM: {route}");
+        }
+
+        public T Deserialize<T>(string json)
+        {
+            var errors = new List<string>();
+            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
+            {
+                Error = delegate (object? sender, Serialization.ErrorEventArgs args)
+                {
+                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
+                    {
+                        errors.Add($"""
+                                    Member: '{args.ErrorContext.Member?.ToString() ?? "<null>"}'
+                                    Path: {args.ErrorContext.Path}
+                                    Error: {args.ErrorContext.Error.Message}
+                                    """);
+                        args.ErrorContext.Handled = true;
+                    }
+                }
+            });
+            if (errors.Count > 0)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
+            }
+            else if (deserialized == null)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
+            }
+            return deserialized;
+        }
+
+        private string GetString(HttpClient client, string route)
+        {
+            var url = GetUrl() + route;
+            Log(url, "");
+            var result = Time.Wait(client.GetAsync(url));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
+        private HttpResponseMessage PostJson<TRequest>(HttpClient client, string route, TRequest body)
+        {
+            var url = GetUrl() + route;
+            using var content = JsonContent.Create(body);
+            Log(url, JsonConvert.SerializeObject(body));
+            return Time.Wait(client.PostAsync(url, content));
+        }
+
+        private string PostJsonString(HttpClient client, string route, string body)
+        {
+            var url = GetUrl() + route;
+            Log(url, body);
+            var content = new StringContent(body);
+            content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+            var result = Time.Wait(client.PostAsync(url, content));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
+        private string GetUrl()
+        {
+            return $"{address.Host}:{address.Port}{baseUrl}";
+        }
+
+        private void Log(string url, string message)
+        {
+            if (logAlias != null)
+            {
+                log.Debug($"({logAlias})({url}) = '{message}'", 3);
+            }
+            else
+            {
+                log.Debug($"({url}) = '{message}'", 3);
+            }
+        }
+    }
+}

Framework/Core/Http.cs

@@ -1,22 +1,13 @@
 using Logging;
-using Newtonsoft.Json;
-using Serialization = Newtonsoft.Json.Serialization;
-using System.Net.Http.Headers;
-using System.Net.Http.Json;
 using Utils;
 
 namespace Core
 {
     public interface IHttp
     {
-        string HttpGetString(string route);
-        T HttpGetJson<T>(string route);
-        TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
-        string HttpPostJson<TRequest>(string route, TRequest body);
-        TResponse HttpPostString<TResponse>(string route, string body);
-        string HttpPostStream(string route, Stream stream);
-        Stream HttpGetStream(string route);
-        T Deserialize<T>(string json);
+        T OnClient<T>(Func<HttpClient, T> action);
+        T OnClient<T>(Func<HttpClient, T> action, string description);
+        IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null);
     }
 
     internal class Http : IHttp
@@ -24,185 +15,44 @@ namespace Core
         private static readonly object httpLock = new object();
         private readonly ILog log;
         private readonly ITimeSet timeSet;
-        private readonly Address address;
-        private readonly string baseUrl;
         private readonly Action<HttpClient> onClientCreated;
-        private readonly string? logAlias;
 
-        internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
-            : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
+        internal Http(ILog log, ITimeSet timeSet)
+            : this(log, timeSet, DoNothing)
         {
         }
 
-        internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
+        internal Http(ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated)
         {
             this.log = log;
             this.timeSet = timeSet;
-            this.address = address;
-            this.baseUrl = baseUrl;
             this.onClientCreated = onClientCreated;
-            this.logAlias = logAlias;
-            if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
-            if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
         }
 
-        public string HttpGetString(string route)
+        public T OnClient<T>(Func<HttpClient, T> action)
         {
+            return OnClient(action, GetDescription());
+        }
+
+        public T OnClient<T>(Func<HttpClient, T> action, string description)
+        {
+            var client = GetClient();
+
             return LockRetry(() =>
             {
-                return GetString(route);
-            }, $"HTTP-GET:{route}");
+                return action(client);
+            }, description);
         }
 
-        public T HttpGetJson<T>(string route)
+        public IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null)
         {
-            return LockRetry(() =>
-            {
-                var json = GetString(route);
-                return Deserialize<T>(json);
-            }, $"HTTP-GET:{route}");
+            return new Endpoint(log, this, address, baseUrl, logAlias);
         }
 
-        public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
+        private string GetDescription()
         {
-            return LockRetry(() =>
-            {
-                var response = PostJson(route, body);
-                var json = Time.Wait(response.Content.ReadAsStringAsync());
-                if (!response.IsSuccessStatusCode)
-                {
-                    throw new HttpRequestException(json);
-                }
-                Log(GetUrl() + route, json);
-                return Deserialize<TResponse>(json);
-            }, $"HTTP-POST-JSON: {route}");
-        }
-
-        public string HttpPostJson<TRequest>(string route, TRequest body)
-        {
-            return LockRetry(() =>
-            {
-                var response = PostJson(route, body);
-                return Time.Wait(response.Content.ReadAsStringAsync());
-            }, $"HTTP-POST-JSON: {route}");
-        }
-
-        public TResponse HttpPostString<TResponse>(string route, string body)
-        {
-            return LockRetry(() =>
-            {
-                var response = PostJsonString(route, body);
-                if (response == null) throw new Exception("Received no response.");
-                var result = Deserialize<TResponse>(response);
-                if (result == null) throw new Exception("Failed to deserialize response");
-                return result;
-            }, $"HTTO-POST-JSON: {route}");
-        }
-
-        public string HttpPostStream(string route, Stream stream)
-        {
-            return LockRetry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, "~ STREAM ~");
-                var content = new StreamContent(stream);
-                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
-                var response = Time.Wait(client.PostAsync(url, content));
-                var str = Time.Wait(response.Content.ReadAsStringAsync());
-                Log(url, str);
-                return str;
-            }, $"HTTP-POST-STREAM: {route}");
-        }
-
-        public Stream HttpGetStream(string route)
-        {
-            return LockRetry(() =>
-            {
-                var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, "~ STREAM ~");
-                return Time.Wait(client.GetStreamAsync(url));
-            }, $"HTTP-GET-STREAM: {route}");
-        }
-
-        public T Deserialize<T>(string json)
-        {
-            var errors = new List<string>();
-            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
-            {
-                Error = delegate(object? sender, Serialization.ErrorEventArgs args)
-                {
-                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
-                    {
-                        errors.Add($"""
-                                    Member: '{args.ErrorContext.Member?.ToString() ?? "<null>"}'
-                                    Path: {args.ErrorContext.Path}
-                                    Error: {args.ErrorContext.Error.Message}
-                                    """);
-                        args.ErrorContext.Handled = true;
-                    }
-                }
-            });
-            if (errors.Count > 0)
-            {
-                throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
-            }
-            else if (deserialized == null)
-            {
-                throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
-            }
-            return deserialized;
-        }
-
-        private string GetString(string route)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            Log(url, "");
-            var result = Time.Wait(client.GetAsync(url));
-            var str = Time.Wait(result.Content.ReadAsStringAsync());
-            Log(url, str);
-            return str;
-        }
-
-        private HttpResponseMessage PostJson<TRequest>(string route, TRequest body)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            using var content = JsonContent.Create(body);
-            Log(url, JsonConvert.SerializeObject(body));
-            return Time.Wait(client.PostAsync(url, content));
-        }
-
-        private string PostJsonString(string route, string body)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            Log(url, body);
-            var content = new StringContent(body);
-            content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
-            var result = Time.Wait(client.PostAsync(url, content));
-            var str = Time.Wait(result.Content.ReadAsStringAsync());
-            Log(url, str);
-            return str;
-        }
-
-        private string GetUrl()
-        {
-            return $"{address.Host}:{address.Port}{baseUrl}";
-        }
-
-        private void Log(string url, string message)
-        {
-            if (logAlias != null)
-            {
-                log.Debug($"({logAlias})({url}) = '{message}'", 3);
-            }
-            else
-            {
-                log.Debug($"({url}) = '{message}'", 3);
-            }
+            // todo: check this:
+            return DebugStack.GetCallerName(skipFrames: 2);
         }
 
         private T LockRetry<T>(Func<T> operation, string description)

Framework/Core/PluginTools.cs

@@ -1,7 +1,6 @@
 using FileUtils;
 using KubernetesWorkflow;
 using Logging;
-using Utils;
 
 namespace Core
 {
@@ -22,9 +21,9 @@ namespace Core
 
     public interface IHttpFactoryTool
     {
-        IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null);
-        IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet timeSet, string? logAlias = null);
-        IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null);
+        IHttp CreateHttp(Action<HttpClient> onClientCreated);
+        IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet);
+        IHttp CreateHttp();
     }
 
     public interface IFileTool
@@ -37,11 +36,11 @@ namespace Core
         private readonly ITimeSet timeSet;
         private readonly WorkflowCreator workflowCreator;
         private readonly IFileManager fileManager;
-        private ILog log;
+        private readonly LogPrefixer log;
 
         internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet)
         {
-            this.log = log;
+            this.log = new LogPrefixer(log);
             this.workflowCreator = workflowCreator;
             this.timeSet = timeSet;
             fileManager = new FileManager(log, fileManagerRootFolder);
@@ -49,22 +48,22 @@ namespace Core
 
         public void ApplyLogPrefix(string prefix)
         {
-            log = new LogPrefixer(log, prefix);
+            log.Prefix = prefix;
         }
 
-        public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
+        public IHttp CreateHttp(Action<HttpClient> onClientCreated)
         {
-            return CreateHttp(address, baseUrl, onClientCreated, timeSet, logAlias);
+            return CreateHttp(onClientCreated, timeSet);
         }
 
-        public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet ts, string? logAlias = null)
+        public IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet ts)
         {
-            return new Http(log, ts, address, baseUrl, onClientCreated, logAlias);
+            return new Http(log, ts, onClientCreated);
         }
 
-        public IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null)
+        public IHttp CreateHttp()
         {
-            return new Http(log, timeSet, address, baseUrl, logAlias);
+            return new Http(log, timeSet);
         }
 
         public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)

Framework/DiscordRewards/CheckConfig.cs

@@ -0,0 +1,21 @@
+using Utils;
+
+namespace DiscordRewards
+{
+    public class CheckConfig
+    {
+        public CheckType Type { get; set; }
+        public ulong MinNumberOfHosts { get; set; }
+        public ByteSize MinSlotSize { get; set; } = 0.Bytes();
+        public TimeSpan MinDuration { get; set; } = TimeSpan.Zero;
+    }
+
+    public enum CheckType
+    {
+        Uninitialized,
+        FilledSlot,
+        FinishedSlot,
+        PostedContract,
+        StartedContract,
+    }
+}

Framework/DiscordRewards/DiscordRewards.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -0,0 +1,13 @@
+namespace DiscordRewards
+{
+    public class GiveRewardsCommand
+    {
+        public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
+    }
+
+    public class RewardUsersCommand
+    {
+        public ulong RewardId { get; set; }
+        public string[] UserAddresses { get; set; } = Array.Empty<string>();
+    }
+}

Framework/DiscordRewards/RewardConfig.cs

@@ -0,0 +1,18 @@
+namespace DiscordRewards
+{
+    public class RewardConfig
+    {
+        public const string UsernameTag = "<USER>";
+
+        public RewardConfig(ulong roleId, string message, CheckConfig checkConfig)
+        {
+            RoleId = roleId;
+            Message = message;
+            CheckConfig = checkConfig;
+        }
+
+        public ulong RoleId { get; }
+        public string Message { get; }
+        public CheckConfig CheckConfig { get; }
+    }
+}

Framework/DiscordRewards/RewardRepo.cs

@@ -0,0 +1,53 @@
+using Utils;
+
+namespace DiscordRewards
+{
+    public class RewardRepo
+    {
+        private static string Tag => RewardConfig.UsernameTag;
+
+        public RewardConfig[] Rewards { get; } = new RewardConfig[]
+        {
+            // Filled any slot
+            new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
+            {
+                Type = CheckType.FilledSlot
+            }),
+
+            // Finished any slot
+            new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
+            {
+                Type = CheckType.FinishedSlot
+            }),
+
+            // Finished a sizable slot
+            new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot!", new CheckConfig
+            {
+                Type = CheckType.FinishedSlot,
+                MinSlotSize = 1.GB(),
+                MinDuration = TimeSpan.FromHours(24.0),
+            }),
+
+            // Posted any contract
+            new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
+            {
+                Type = CheckType.PostedContract
+            }),
+
+            // Started any contract
+            new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
+            {
+                Type = CheckType.StartedContract
+            }),
+
+            // Started a sizable contract
+            new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time!", new CheckConfig
+            {
+                Type = CheckType.FinishedSlot,
+                MinNumberOfHosts = 4,
+                MinSlotSize = 1.GB(),
+                MinDuration = TimeSpan.FromHours(24.0),
+            })
+        };
+    }
+}

Framework/FileUtils/FileManager.cs

@@ -45,7 +45,7 @@ namespace FileUtils
         {
             var sw = Stopwatch.Begin(log);
             var result = GenerateRandomFile(size, label);
-            sw.End($"Generated file '{result.Describe()}'.");
+            sw.End($"Generated file {result.Describe()}.");
             return result;
         }
 

Framework/FileUtils/TrackedFile.cs

@@ -37,6 +37,11 @@ namespace FileUtils
             return $"'{Filename}'{sizePostfix}";
         }
 
+        public ByteSize GetFilesize()
+        {
+            return new ByteSize(GetFileSize());
+        }
+
         private void AssertEqual(TrackedFile? actual)
         {
             if (actual == null)  FrameworkAssert.Fail("TestFile is null.");

Framework/GethConnector/GethConnector.cs

@@ -0,0 +1,39 @@
+using CodexContractsPlugin;
+using GethPlugin;
+using Logging;
+
+namespace GethConnector
+{
+    public class GethConnector
+    {
+        public IGethNode GethNode { get; }
+        public ICodexContracts CodexContracts { get; }
+
+        public static GethConnector? Initialize(ILog log)
+        {
+            if (!string.IsNullOrEmpty(GethInput.LoadError))
+            {
+                var msg = "Geth input incorrect: " + GethInput.LoadError;
+                log.Error(msg);
+                return null;
+            }
+
+            var contractsDeployment = new CodexContractsDeployment(
+                marketplaceAddress: GethInput.MarketplaceAddress,
+                abi: GethInput.ABI,
+                tokenAddress: GethInput.TokenAddress
+            );
+
+            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+            var contracts = new CodexContractsAccess(log, gethNode, contractsDeployment);
+
+            return new GethConnector(gethNode, contracts);
+        }
+
+        private GethConnector(IGethNode gethNode, ICodexContracts codexContracts)
+        {
+            GethNode = gethNode;
+            CodexContracts = codexContracts;
+        }
+    }
+}

Framework/GethConnector/GethConnector.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\GethPlugin\GethPlugin.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/GethConnector/GethInput.cs

@@ -0,0 +1,52 @@
+namespace GethConnector
+{
+    public static class GethInput
+    {
+        private const string GethHostVar = "GETH_HOST";
+        private const string GethPortVar = "GETH_HTTP_PORT";
+        private const string GethPrivKeyVar = "GETH_PRIVATE_KEY";
+        private const string MarketplaceAddressVar = "CODEXCONTRACTS_MARKETPLACEADDRESS";
+        private const string TokenAddressVar = "CODEXCONTRACTS_TOKENADDRESS";
+        private const string AbiVar = "CODEXCONTRACTS_ABI";
+
+        static GethInput()
+        {
+            var error = new List<string>();
+            var gethHost = GetEnvVar(error, GethHostVar);
+            var gethPort = Convert.ToInt32(GetEnvVar(error, GethPortVar));
+            var privateKey = GetEnvVar(error, GethPrivKeyVar);
+            var marketplaceAddress = GetEnvVar(error, MarketplaceAddressVar);
+            var tokenAddress = GetEnvVar(error, TokenAddressVar);
+            var abi = GetEnvVar(error, AbiVar);
+
+            if (error.Any())
+            {
+                LoadError = string.Join(", ", error);
+            }
+            else
+            {
+                GethHost = gethHost!;
+                GethPort = gethPort;
+                PrivateKey = privateKey!;
+                MarketplaceAddress = marketplaceAddress!;
+                TokenAddress = tokenAddress!;
+                ABI = abi!;
+            }
+        }
+
+        public static string GethHost { get; } = string.Empty;
+        public static int GethPort { get; }
+        public static string PrivateKey { get; } = string.Empty;
+        public static string MarketplaceAddress { get; } = string.Empty;
+        public static string TokenAddress { get; } = string.Empty;
+        public static string ABI { get; } = string.Empty;
+        public static string LoadError { get; } = string.Empty;
+
+        private static string? GetEnvVar(List<string> error, string name)
+        {
+            var result = Environment.GetEnvironmentVariable(name);
+            if (string.IsNullOrEmpty(result)) error.Add($"'{name}' is not set.");
+            return result;
+        }
+    }
+}

Framework/KubernetesWorkflow/CommandRunner.cs

@@ -6,17 +6,17 @@ namespace KubernetesWorkflow
     {
         private readonly K8sClient client;
         private readonly string k8sNamespace;
-        private readonly RunningPod pod;
+        private readonly string podName;
         private readonly string containerName;
         private readonly string command;
         private readonly string[] arguments;
         private readonly List<string> lines = new List<string>();
 
-        public CommandRunner(K8sClient client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
+        public CommandRunner(K8sClient client, string k8sNamespace, string podName, string containerName, string command, string[] arguments)
         {
             this.client = client;
             this.k8sNamespace = k8sNamespace;
-            this.pod = pod;
+            this.podName = podName;
             this.containerName = containerName;
             this.command = command;
             this.arguments = arguments;
@@ -27,7 +27,7 @@ namespace KubernetesWorkflow
             var input = new[] { command }.Concat(arguments).ToArray();
 
             Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
-                pod.PodInfo.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
+                podName, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
         }
 
         public string GetStdOut()

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -7,19 +7,23 @@ namespace KubernetesWorkflow
     {
         private readonly ILog log;
         private readonly KubernetesClientConfiguration config;
+        private readonly string containerName;
+        private readonly string podName;
+        private readonly string recipeName;
         private readonly string k8sNamespace;
-        private readonly RunningContainer container;
         private ILogHandler? logHandler;
         private CancellationTokenSource cts;
         private Task? worker;
         private Exception? workerException;
 
-        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container)
+        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
         {
             this.log = log;
             this.config = config;
+            this.containerName = containerName;
+            this.podName = podName;
+            this.recipeName = recipeName;
             this.k8sNamespace = k8sNamespace;
-            this.container = container;
             cts = new CancellationTokenSource();
         }
 
@@ -46,7 +50,7 @@ namespace KubernetesWorkflow
         public bool HasContainerCrashed()
         {
             using var client = new Kubernetes(config);
-            return HasContainerBeenRestarted(client, container.Pod.PodInfo.Name);
+            return HasContainerBeenRestarted(client);
         }
 
         private void Worker()
@@ -66,29 +70,26 @@ namespace KubernetesWorkflow
             using var client = new Kubernetes(config);
             while (!token.IsCancellationRequested)
             {
-                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(1));
+                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10));
 
-                var pod = container.Pod;
-                var recipe = container.Recipe;
-                var podName = pod.PodInfo.Name;
-                if (HasContainerBeenRestarted(client, podName))
+                if (HasContainerBeenRestarted(client))
                 {
-                    DownloadCrashedContainerLogs(client, podName, recipe);
+                    DownloadCrashedContainerLogs(client);
                     return;
                 }
             }
         }
 
-        private bool HasContainerBeenRestarted(Kubernetes client, string podName)
+        private bool HasContainerBeenRestarted(Kubernetes client)
         {
             var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
             return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
         }
 
-        private void DownloadCrashedContainerLogs(Kubernetes client, string podName, ContainerRecipe recipe)
+        private void DownloadCrashedContainerLogs(Kubernetes client)
         {
-            log.Log("Pod crash detected for " + container.Name);
-            using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipe.Name, previous: true);
+            log.Log("Pod crash detected for " + containerName);
+            using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
             logHandler!.Log(stream);
         }
     }

Framework/KubernetesWorkflow/K8sCluster.cs

@@ -54,16 +54,4 @@ namespace KubernetesWorkflow
             }
         }
     }
-
-    public class K8sNodeLabel
-    {
-        public K8sNodeLabel(string key, string value)
-        {
-            Key = key;
-            Value = value;
-        }
-
-        public string Key { get; }
-        public string Value { get; }
-    }
 }

Framework/KubernetesWorkflow/K8sController.cs

@@ -1,5 +1,7 @@
 using k8s;
 using k8s.Models;
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
 using Logging;
 using Utils;
 
@@ -11,7 +13,7 @@ namespace KubernetesWorkflow
         private readonly K8sCluster cluster;
         private readonly WorkflowNumberSource workflowNumberSource;
         private readonly K8sClient client;
-        private const string podLabelKey = "pod-uuid";
+        public const string PodLabelKey = "pod-uuid";
 
         public K8sController(ILog log, K8sCluster cluster, WorkflowNumberSource workflowNumberSource, string k8sNamespace)
         {
@@ -28,57 +30,54 @@ namespace KubernetesWorkflow
             client.Dispose();
         }
 
-        public RunningPod BringOnline(ContainerRecipe[] containerRecipes, ILocation location)
+        public StartResult BringOnline(ContainerRecipe[] containerRecipes, ILocation location)
         {
             log.Debug();
             EnsureNamespace();
 
             var podLabel = K8sNameUtils.Format(Guid.NewGuid().ToString());
-            var deploymentName = CreateDeployment(containerRecipes, location, podLabel);
-            var (serviceName, servicePortsMap) = CreateService(containerRecipes);
+            var deployment = CreateDeployment(containerRecipes, location, podLabel);
+            var internalService = CreateInternalService(containerRecipes);
+            var externalService = CreateExternalService(containerRecipes);
 
-            var pod = FindPodByLabel(podLabel);
-            var podInfo = CreatePodInfo(pod);
-
-            return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray());
+            return new StartResult(cluster, containerRecipes, deployment, internalService, externalService);
         }
 
-        private V1Pod FindPodByLabel(string podLabel)
+        public PodInfo GetPodInfo(RunningDeployment deployment)
         {
-            var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace));
-            foreach (var pod in pods.Items)
-            {
-                var label = pod.GetLabel(podLabelKey);
-                if (label == podLabel)
-                {
-                    return pod;
-                }
-            }
-            throw new Exception("Unable to find pod by label.");
+            var pod = GetPodForDeployment(deployment);
+            return CreatePodInfo(pod);
         }
 
-        public void Stop(RunningPod pod)
+        public void Stop(StartResult startResult, bool waitTillStopped)
         {
             log.Debug();
-            if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName);
-            DeleteDeployment(pod.DeploymentName);
-            WaitUntilDeploymentOffline(pod.DeploymentName);
-            WaitUntilPodOffline(pod.PodInfo.Name);
+            if (startResult.InternalService != null) DeleteService(startResult.InternalService);
+            if (startResult.ExternalService != null) DeleteService(startResult.ExternalService);
+            DeleteDeployment(startResult.Deployment);
+
+            if (waitTillStopped) WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
         }
 
-        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler, int? tailLines)
+        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
         {
             log.Debug();
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sNamespace, recipe.Name, tailLines: tailLines));
+
+            var podName = GetPodName(container);
+            var recipeName = container.Recipe.Name;
+
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines));
             logHandler.Log(stream);
         }
 
-        public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
+        public string ExecuteCommand(RunningContainer container, string command, params string[] args)
         {
+            var containerName = container.Recipe.Name;
             var cmdAndArgs = $"{containerName}: {command} ({string.Join(",", args)})";
             log.Debug(cmdAndArgs);
 
-            var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
+            var podName = GetPodName(container);
+            var runner = new CommandRunner(client, K8sNamespace, podName, containerName, command, args);
             runner.Run();
             var result = runner.GetStdOut();
 
@@ -86,6 +85,31 @@ namespace KubernetesWorkflow
             return result;
         }
 
+        public int[] GetUsedExternalPorts()
+        {
+            return client.Run(c =>
+            {
+                var result = new List<int>();
+
+                var services = c.ListServiceForAllNamespaces();
+                var nodePorts = services.Items.Where(s => s.Spec.Type == "NodePort").ToArray();
+                if (!nodePorts.Any()) return result.ToArray();
+
+                foreach (var service in nodePorts)
+                {
+                    foreach (var port in service.Spec.Ports)
+                    {
+                        if (port.NodePort.HasValue)
+                        {
+                            result.Add(port.NodePort.Value);
+                        }
+                    }
+                }
+
+                return result.ToArray();
+            });
+        }
+
         public void DeleteAllNamespacesStartingWith(string prefix)
         {
             log.Debug();
@@ -315,7 +339,7 @@ namespace KubernetesWorkflow
 
         #region Deployment management
 
-        private string CreateDeployment(ContainerRecipe[] containerRecipes, ILocation location, string podLabel)
+        private RunningDeployment CreateDeployment(ContainerRecipe[] containerRecipes, ILocation location, string podLabel)
         {
             var deploymentSpec = new V1Deployment
             {
@@ -337,6 +361,8 @@ namespace KubernetesWorkflow
                         },
                         Spec = new V1PodSpec
                         {
+                            PriorityClassName = GetPriorityClassName(containerRecipes),
+                            Affinity = CreatePodAffinity(containerRecipes),
                             NodeSelector = CreateNodeSelector(location),
                             Containers = CreateDeploymentContainers(containerRecipes),
                             Volumes = CreateVolumes(containerRecipes)
@@ -348,13 +374,14 @@ namespace KubernetesWorkflow
             client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
             WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
 
-            return deploymentSpec.Metadata.Name;
+            var name = deploymentSpec.Metadata.Name;
+            return new RunningDeployment(name, podLabel);
         }
 
-        private void DeleteDeployment(string deploymentName)
+        private void DeleteDeployment(RunningDeployment deployment)
         {
-            client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sNamespace));
-            WaitUntilDeploymentOffline(deploymentName);
+            client.Run(c => c.DeleteNamespacedDeployment(deployment.Name, K8sNamespace));
+            WaitUntilDeploymentOffline(deployment.Name);
         }
 
         private IDictionary<string, string> CreateNodeSelector(ILocation location)
@@ -368,12 +395,57 @@ namespace KubernetesWorkflow
             };
         }
 
+        private V1Affinity? CreatePodAffinity(ContainerRecipe[] recipes)
+        {
+            var notIns = recipes
+                .Select(r => r.SchedulingAffinity.NotIn)
+                .Where(n => !string.IsNullOrEmpty(n))
+                .Distinct()
+                .ToList();
+
+            if (!notIns.Any()) return null;
+
+            return new V1Affinity
+            {
+                NodeAffinity = new V1NodeAffinity
+                {
+                    RequiredDuringSchedulingIgnoredDuringExecution = new V1NodeSelector
+                    {
+                        NodeSelectorTerms = new List<V1NodeSelectorTerm>
+                        { 
+                            new V1NodeSelectorTerm
+                            {
+                                MatchExpressions = new List<V1NodeSelectorRequirement>
+                                {
+                                    new V1NodeSelectorRequirement
+                                    {
+                                        Key = "allow-tests-pods",
+                                        OperatorProperty = "NotIn",
+                                        Values = notIns
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            };
+        }
+
         private K8sNodeLabel? GetNodeLabelForLocation(ILocation location)
         {
             var l = (Location)location;
             return l.NodeLabel;
         }
 
+        private string GetPriorityClassName(ContainerRecipe[] containerRecipes)
+        {
+            if (containerRecipes.Any(c => c.SetCriticalPriority))
+            {
+                return "system-node-critical";
+            }
+            return null!;
+        }
+
         private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes)
         {
             return containerRecipes.First().PodLabels.GetLabels();
@@ -382,7 +454,7 @@ namespace KubernetesWorkflow
         private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes, string podLabel)
         {
             var labels = containerRecipes.First().PodLabels.Clone();
-            labels.Add(podLabelKey, podLabel);
+            labels.Add(PodLabelKey, podLabel);
             return labels.GetLabels();
         }
 
@@ -427,10 +499,17 @@ namespace KubernetesWorkflow
                 Ports = CreateContainerPorts(recipe),
                 Env = CreateEnv(recipe),
                 VolumeMounts = CreateContainerVolumeMounts(recipe),
-                Resources = CreateResourceLimits(recipe)
+                Resources = CreateResourceLimits(recipe),
+                Command = CreateCommandList(recipe)
             };
         }
 
+        private IList<string> CreateCommandList(ContainerRecipe recipe)
+        {
+            if (recipe.CommandOverride == null || !recipe.CommandOverride.Command.Any()) return null!;
+            return recipe.CommandOverride.Command.ToList();
+        }
+
         private V1ResourceRequirements CreateResourceLimits(ContainerRecipe recipe)
         {
             return new V1ResourceRequirements
@@ -609,118 +688,174 @@ namespace KubernetesWorkflow
 
         private string GetNameForPort(ContainerRecipe recipe, Port port)
         {
-            return $"p{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}-{port.Protocol.ToString().ToLowerInvariant()}";
+            var inputs = new[]
+            {
+                $"p{workflowNumberSource.WorkflowNumber}",
+                recipe.Number.ToString(),
+                port.Number.ToString(),
+                port.Protocol.ToString().ToLowerInvariant()
+            };
+
+            return K8sNameUtils.FormatPortName(string.Join(",", inputs));
+        }
+
+        private string GetPodName(RunningContainer container)
+        {
+            return GetPodForDeployment(container.RunningContainers.StartResult.Deployment).Metadata.Name;
+        }
+
+        private V1Pod GetPodForDeployment(RunningDeployment deployment)
+        {
+            return Time.Retry(() => GetPodForDeplomentInternal(deployment),
+                // We will wait up to 1 minute, k8s might be moving pods around.
+                maxRetries: 6,
+                retryTime: TimeSpan.FromSeconds(10),
+                description: "Find pod by label for deployment.");
+        }
+
+        private V1Pod GetPodForDeplomentInternal(RunningDeployment deployment)
+        {
+            var allPods = client.Run(c => c.ListNamespacedPod(K8sNamespace));
+            var pods = allPods.Items.Where(p => p.GetLabel(PodLabelKey) == deployment.PodLabel).ToArray();
+
+            if (pods.Length != 1)
+            {
+                var allLabels = allPods.Items.Select(p =>
+                {
+                    var labels = string.Join(",", p.Labels().Select(l => $"{l.Key}={l.Value}"));
+                    return $"pod:'{p.Name()}' has labels: [{labels}]";
+                });
+                throw new Exception($"Expected to find 1 pod by podLabel '{deployment.PodLabel}'. Found: {pods.Length}. " +
+                    $"Total number of pods: {allPods.Items.Count}. Their labels: {string.Join(Environment.NewLine, allLabels)}");
+            }
+            return pods[0];
         }
 
         #endregion
 
         #region Service management
 
-        private (string, List<ContainerRecipePortMapEntry>) CreateService(ContainerRecipe[] containerRecipes)
+        private RunningService? CreateInternalService(ContainerRecipe[] recipes)
         {
-            var result = new List<ContainerRecipePortMapEntry>();
+            return CreateService(recipes, r => r.InternalPorts.Concat(r.ExposedPorts).ToArray(), "ClusterIP", "int", false);
+        }
 
-            var ports = CreateServicePorts(containerRecipes);
+        private RunningService? CreateExternalService(ContainerRecipe[] recipes)
+        {
+            return CreateService(recipes, r => r.ExposedPorts, "NodePort", "ext", true);
+        }
 
-            if (!ports.Any())
-            {
-                // None of these container-recipes wish to expose anything via a service port.
-                // So, we don't have to create a service.
-                return (string.Empty, result);
-            }
+        private RunningService? CreateService(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector, string serviceType, string namePostfix, bool isNodePort)
+        {
+            var ports = CreateServicePorts(recipes, portSelector, isNodePort);
+            if (!ports.Any()) return null;
 
             var serviceSpec = new V1Service
             {
                 ApiVersion = "v1",
-                Metadata = CreateServiceMetadata(),
+                Metadata = CreateServiceMetadata(recipes, namePostfix),
                 Spec = new V1ServiceSpec
                 {
-                    Type = "NodePort",
-                    Selector = GetSelector(containerRecipes),
-                    Ports = ports
+                    Type = serviceType,
+                    Selector = GetSelector(recipes),
+                    Ports = ports,
                 }
             };
 
             client.Run(c => c.CreateNamespacedService(serviceSpec, K8sNamespace));
 
-            ReadBackServiceAndMapPorts(serviceSpec, containerRecipes, result);
+            var result = ReadBackServiceAndMapPorts(serviceSpec, recipes);
+            var name = serviceSpec.Metadata.Name;
 
-            return (serviceSpec.Metadata.Name, result);
+            return new RunningService(name, result);
         }
 
-        private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, List<ContainerRecipePortMapEntry> result)
+        private List<ContainerRecipePortMapEntry> ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes)
         {
-            // For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
+            var result = new List<ContainerRecipePortMapEntry>();
+
+            // For each container-recipe-port, we need to figure out which service-ports it was assigned by K8s.
             var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sNamespace));
             foreach (var r in containerRecipes)
             {
-                foreach (var port in r.ExposedPorts)
+                var recipePorts = r.ExposedPorts.Concat(r.InternalPorts).ToArray();
+                foreach (var port in recipePorts)
                 {
                     var portName = GetNameForPort(r, port);
 
                     var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName);
-                    if (matchingServicePorts.Any())
-                    {
-                        // These service ports belongs to this recipe.
-                        var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, port.Tag, port.Protocol));
-                        var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
+                    var ports = matchingServicePorts.Select(p => MapPortIfAble(p, port.Tag, port.Protocol)).ToArray();
 
+                    if (ports.Any())
+                    {
                         result.Add(new ContainerRecipePortMapEntry(r.Number, ports));
+                        log.Debug($"Service Readback: {portName} found: {string.Join(",", ports.Select(p => p.ToString()))}");
                     }
                 }
             }
+
+            return result;
         }
 
-        private Port? MapNodePortIfAble(V1ServicePort p, string tag, PortProtocol protocol)
+        private Port MapPortIfAble(V1ServicePort p, string tag, PortProtocol protocol)
         {
-            if (p.NodePort == null) return null;
-            return new Port(p.NodePort.Value, tag, protocol);
+            if (p.NodePort != null) return new Port(p.NodePort.Value, tag, protocol);
+            if (p.Port > 0) return new Port(p.Port, tag, protocol);
+            throw new Exception("Unable to map port.");
         }
 
-        private void DeleteService(string serviceName)
+        private void DeleteService(RunningService service)
         {
-            client.Run(c => c.DeleteNamespacedService(serviceName, K8sNamespace));
+            client.Run(c => c.DeleteNamespacedService(service.Name, K8sNamespace));
         }
 
-        private V1ObjectMeta CreateServiceMetadata()
+        private V1ObjectMeta CreateServiceMetadata(ContainerRecipe[] containerRecipes, string namePostfix)
         {
+            var recipeName = containerRecipes.First().Name;
+            var name = K8sNameUtils.Format($"{recipeName}-{workflowNumberSource.WorkflowNumber}-{namePostfix}");
+            log.Debug("Creating service: " + name);
             return new V1ObjectMeta
             {
-                Name = "service-" + workflowNumberSource.WorkflowNumber,
-                NamespaceProperty = K8sNamespace
+                Name = name,
+                NamespaceProperty = K8sNamespace,
             };
         }
 
-        private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes)
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector, bool isNodePort)
         {
             var result = new List<V1ServicePort>();
             foreach (var recipe in recipes)
             {
-                result.AddRange(CreateServicePorts(recipe));
+                var ports = portSelector(recipe);
+                foreach (var port in ports)
+                {
+                    result.AddRange(CreateServicePorts(recipe, port, isNodePort));
+                }
             }
             return result;
         }
 
-        private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe)
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe, Port recipePort, bool isNodePort)
         {
             var result = new List<V1ServicePort>();
-            foreach (var port in recipe.ExposedPorts)
-            {
-                if (port.IsTcp()) CreateServicePort(result, recipe, port, "TCP");
-                if (port.IsUdp()) CreateServicePort(result, recipe, port, "UDP");
-            }
+            if (recipePort.IsTcp()) CreateServicePort(result, recipe, recipePort, "TCP", isNodePort);
+            if (recipePort.IsUdp()) CreateServicePort(result, recipe, recipePort, "UDP", isNodePort);
             return result;
         }
 
-        private void CreateServicePort(List<V1ServicePort> result, ContainerRecipe recipe, Port port, string protocol)
+        private void CreateServicePort(List<V1ServicePort> result, ContainerRecipe recipe, Port port, string protocol, bool isNodePort)
         {
-            result.Add(new V1ServicePort
+            var p = new V1ServicePort
             {
                 Name = GetNameForPort(recipe, port),
                 Protocol = protocol,
                 Port = port.Number,
-                TargetPort = GetNameForPort(recipe, port),
-            });
+                TargetPort = GetNameForPort(recipe, port)
+            };
+
+            if (isNodePort) p.NodePort = port.Number;
+
+            result.Add(p);
         }
 
         #endregion
@@ -751,13 +886,12 @@ namespace KubernetesWorkflow
             });
         }
 
-        private void WaitUntilPodOffline(string podName)
+        private void WaitUntilPodsForDeploymentAreOffline(RunningDeployment deployment)
         {
             WaitUntil(() =>
             {
-                var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)).Items;
-                var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
-                return pod == null;
+                var pods = FindPodsByLabel(deployment.PodLabel);
+                return !pods.Any();
             });
         }
 
@@ -778,7 +912,17 @@ namespace KubernetesWorkflow
 
         public CrashWatcher CreateCrashWatcher(RunningContainer container)
         {
-            return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sNamespace, container);
+            var containerName = container.Name;
+            var podName = GetPodName(container);
+            var recipeName = container.Recipe.Name;
+
+            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
+        }
+
+        private V1Pod[] FindPodsByLabel(string podLabel)
+        {
+            var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace));
+            return pods.Items.Where(p => p.GetLabel(PodLabelKey) == podLabel).ToArray();
         }
 
         private PodInfo CreatePodInfo(V1Pod pod)

Framework/KubernetesWorkflow/K8sHooks.cs

@@ -1,4 +1,7 @@
-namespace KubernetesWorkflow
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
+
+namespace KubernetesWorkflow
 {
     public interface IK8sHooks
     {

Framework/KubernetesWorkflow/K8sNameUtils.cs

@@ -3,8 +3,19 @@
     public static class K8sNameUtils
     {
         public static string Format(string s)
+        {
+            return Format(s, 62);
+        }
+
+        public static string FormatPortName(string s)
+        {
+            return Format(s, 15);
+        }
+
+        private static string Format(string s, int maxLength)
         {
             var result = s.ToLowerInvariant()
+                .Replace("_", "-")
                 .Replace(" ", "-")
                 .Replace(":", "-")
                 .Replace("/", "-")
@@ -14,7 +25,7 @@
                 .Replace(",", "-");
 
             result = result.Trim('-');
-            if (result.Length > 62) result = result.Substring(0, 62);
+            if (result.Length > maxLength) result = result.Substring(0, maxLength);
 
             return result;
         }

Framework/KubernetesWorkflow/Location.cs

@@ -1,4 +1,6 @@
-namespace KubernetesWorkflow
+using KubernetesWorkflow.Types;
+
+namespace KubernetesWorkflow
 {
     public interface ILocation
     {

Framework/KubernetesWorkflow/LocationProvider.cs

@@ -1,4 +1,5 @@
-using Logging;
+using KubernetesWorkflow.Types;
+using Logging;
 
 namespace KubernetesWorkflow
 {

Framework/KubernetesWorkflow/LogHandler.cs

@@ -0,0 +1,23 @@
+namespace KubernetesWorkflow
+{
+    public interface ILogHandler
+    {
+        void Log(Stream log);
+    }
+
+    public abstract class LogHandler : ILogHandler
+    {
+        public void Log(Stream log)
+        {
+            using var reader = new StreamReader(log);
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                ProcessLine(line);
+                line = reader.ReadLine();
+            }
+        }
+
+        protected abstract void ProcessLine(string line);
+    }
+}

Framework/KubernetesWorkflow/PublicIpService.cs

@@ -0,0 +1,7 @@
+namespace KubernetesWorkflow
+{
+    public static class PublicIpService
+    {
+        public static string Address { get; } = "ip.codex.storage";
+    }
+}

Framework/KubernetesWorkflow/Recipe/CommandOverride.cs

@@ -0,0 +1,12 @@
+namespace KubernetesWorkflow.Recipe
+{
+    public class CommandOverride
+    {
+        public CommandOverride(params string[] command)
+        {
+            Command = command;
+        }
+
+        public string[] Command { get; }
+    }
+}

Framework/KubernetesWorkflow/Recipe/ContainerAdditionals.cs

@@ -1,6 +1,6 @@
 using Newtonsoft.Json.Linq;
 
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class ContainerAdditionals
     {
@@ -20,7 +20,7 @@ namespace KubernetesWorkflow
         {
             var typeName = GetTypeName(typeof(T));
             var userData = Additionals.SingleOrDefault(a => a.Type == typeName);
-            if (userData == null) return default(T);
+            if (userData == null) return default;
             var jobject = (JObject)userData.UserData;
             return jobject.ToObject<T>();
         }

Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs

@@ -1,13 +1,16 @@
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class ContainerRecipe
     {
-        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
+        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, CommandOverride commandOverride, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
         {
             Number = number;
             NameOverride = nameOverride;
             Image = image;
             Resources = resources;
+            SchedulingAffinity = schedulingAffinity;
+            CommandOverride = commandOverride;
+            SetCriticalPriority = setCriticalPriority;
             ExposedPorts = exposedPorts;
             InternalPorts = internalPorts;
             EnvVars = envVars;
@@ -32,6 +35,9 @@
         public int Number { get; }
         public string? NameOverride { get; }
         public ContainerResources Resources { get; }
+        public SchedulingAffinity SchedulingAffinity { get; }
+        public CommandOverride CommandOverride { get; }
+        public bool SetCriticalPriority { get; }
         public string Image { get; }
         public Port[] ExposedPorts { get; }
         public Port[] InternalPorts { get; }
@@ -53,6 +59,7 @@
                 $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
                 $"envVars: {string.Join(",", EnvVars.Select(v => v.ToString()))}, " +
                 $"limits: {Resources}, " +
+                $"affinity: {SchedulingAffinity}, " +
                 $"volumes: {string.Join(",", Volumes.Select(v => $"'{v.MountPath}'"))}";
         }
     }
@@ -64,6 +71,11 @@
             Number = number;
             Tag = tag;
             Protocol = protocol;
+
+            if (string.IsNullOrWhiteSpace(Tag))
+            {
+                throw new Exception("A unique port tag is required");
+            }
         }
 
         public int Number { get; }

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -1,6 +1,6 @@
 using Utils;
 
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public abstract class ContainerRecipeFactory
     {
@@ -13,6 +13,9 @@ namespace KubernetesWorkflow
         private readonly List<object> additionals = new List<object>();
         private RecipeComponentFactory factory = null!;
         private ContainerResources resources = new ContainerResources();
+        private SchedulingAffinity schedulingAffinity = new SchedulingAffinity();
+        private CommandOverride commandOverride = new CommandOverride();
+        private bool setCriticalPriority;
 
         public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config)
         {
@@ -22,7 +25,7 @@ namespace KubernetesWorkflow
 
             Initialize(config);
 
-            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources,
+            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, commandOverride, setCriticalPriority,
                 exposedPorts.ToArray(),
                 internalPorts.ToArray(),
                 envVars.ToArray(),
@@ -40,6 +43,9 @@ namespace KubernetesWorkflow
             additionals.Clear();
             this.factory = null!;
             resources = new ContainerResources();
+            schedulingAffinity = new SchedulingAffinity();
+            commandOverride = new CommandOverride();
+            setCriticalPriority = false;
 
             return recipe;
         }
@@ -52,17 +58,17 @@ namespace KubernetesWorkflow
 
         protected Port AddExposedPort(string tag, PortProtocol protocol = PortProtocol.TCP)
         {
-            return AddExposedPort(factory.CreatePort(tag, protocol));
+            return AddExposedPort(factory.CreateExternalPort(tag, protocol));
         }
 
         protected Port AddExposedPort(int number, string tag, PortProtocol protocol = PortProtocol.TCP)
         {
-            return AddExposedPort(factory.CreatePort(number, tag, protocol));
+            return AddExposedPort(factory.CreateExternalPort(number, tag, protocol));
         }
 
         protected Port AddInternalPort(string tag = "", PortProtocol protocol = PortProtocol.TCP)
         {
-            var p = factory.CreatePort(tag, protocol);
+            var p = factory.CreateInternalPort(tag, protocol);
             internalPorts.Add(p);
             return p;
         }
@@ -121,11 +127,28 @@ namespace KubernetesWorkflow
             SetResourcesRequest(new ContainerResourceSet(milliCPUs, memory));
         }
 
-        protected void SetResourceLimits(int milliCPUs, ByteSize memory)
+        protected void SetSchedulingAffinity(string notIn)
         {
-            SetResourceLimits(new ContainerResourceSet(milliCPUs, memory));
+            schedulingAffinity = new SchedulingAffinity(notIn);
         }
 
+        protected void OverrideCommand(params string[] command)
+        {
+            commandOverride = new CommandOverride(command);
+        }
+
+        protected void SetSystemCriticalPriority()
+        {
+            setCriticalPriority = true;
+        }
+
+        // Disabled following a possible bug in the k8s cluster that will throttle containers much more than is
+        // called for if they have resource limits defined.
+        //protected void SetResourceLimits(int milliCPUs, ByteSize memory)
+        //{
+        //    SetResourceLimits(new ContainerResourceSet(milliCPUs, memory));
+        //}
+
         protected void SetResourcesRequest(ContainerResourceSet requests)
         {
             resources.Requests = requests;

Framework/KubernetesWorkflow/Recipe/ContainerResources.cs

@@ -1,6 +1,6 @@
 using Utils;
 
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class ContainerResources
     {

Framework/KubernetesWorkflow/Recipe/PodAnnotations.cs

@@ -1,4 +1,4 @@
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class PodAnnotations
     {

Framework/KubernetesWorkflow/Recipe/PodLabels.cs

@@ -1,4 +1,4 @@
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class PodLabels
     {

Framework/KubernetesWorkflow/Recipe/RecipeComponentFactory.cs

@@ -0,0 +1,50 @@
+using System.Globalization;
+using Utils;
+
+namespace KubernetesWorkflow.Recipe
+{
+    public class RecipeComponentFactory
+    {
+        private readonly NumberSource internalNumberSource = new NumberSource(8080);
+        private static readonly NumberSource externalNumberSource = new NumberSource(30000);
+        private static int[] usedExternalPorts = Array.Empty<int>();
+
+        public void Update(K8sController controller)
+        {
+            usedExternalPorts = controller.GetUsedExternalPorts();
+        }
+
+        public Port CreateInternalPort(string tag, PortProtocol protocol)
+        {
+            return new Port(internalNumberSource.GetNextNumber(), tag, protocol);
+        }
+
+        public Port CreateExternalPort(int number, string tag, PortProtocol protocol)
+        {
+            if (usedExternalPorts.Contains(number)) throw new Exception($"External port number {number} is already in use by the cluster.");
+            return new Port(number, tag, protocol);
+        }
+
+        public Port CreateExternalPort(string tag, PortProtocol protocol)
+        {
+            while (true)
+            {
+                var number = externalNumberSource.GetNextNumber();
+                if (!usedExternalPorts.Contains(number))
+                {
+                    return new Port(number, tag, protocol);
+                }
+            }
+        }
+
+        public EnvVar CreateEnvVar(string name, int value)
+        {
+            return CreateEnvVar(name, value.ToString(CultureInfo.InvariantCulture));
+        }
+
+        public EnvVar CreateEnvVar(string name, string value)
+        {
+            return new EnvVar(name, value);
+        }
+    }
+}

Framework/KubernetesWorkflow/Recipe/SchedulingAffinity.cs

@@ -0,0 +1,18 @@
+namespace KubernetesWorkflow.Recipe
+{
+    public class SchedulingAffinity
+    {
+        public SchedulingAffinity(string? notIn = null)
+        {
+            NotIn = notIn;
+        }
+
+        public string? NotIn { get; }
+
+        public override string ToString()
+        {
+            if (string.IsNullOrEmpty(NotIn)) return "none";
+            return "notIn:" + NotIn;
+        }
+    }
+}

Framework/KubernetesWorkflow/RecipeComponentFactory.cs

@@ -1,30 +0,0 @@
-using System.Globalization;
-using Utils;
-
-namespace KubernetesWorkflow
-{
-    public class RecipeComponentFactory
-    {
-        private NumberSource portNumberSource = new NumberSource(8080);
-
-        public Port CreatePort(int number, string tag, PortProtocol protocol)
-        {
-            return new Port(number, tag, protocol);
-        }
-
-        public Port CreatePort(string tag, PortProtocol protocol)
-        {
-            return new Port(portNumberSource.GetNextNumber(), tag, protocol);
-        }
-
-        public EnvVar CreateEnvVar(string name, int value)
-        {
-            return CreateEnvVar(name, value.ToString(CultureInfo.InvariantCulture));
-        }
-
-        public EnvVar CreateEnvVar(string name, string value)
-        {
-            return new EnvVar(name, value);
-        }
-    }
-}

Framework/KubernetesWorkflow/RunnerLocationUtils.cs

@@ -1,63 +1,38 @@
-using System.Net.NetworkInformation;
-using Utils;
-
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow
 {
     internal enum RunnerLocation
     {
+        Unknown,
         ExternalToCluster,
         InternalToCluster,
     }
 
     internal static class RunnerLocationUtils
     {
-        private static RunnerLocation? knownLocation = null;
+        private static RunnerLocation location = RunnerLocation.Unknown;
 
-        internal static RunnerLocation DetermineRunnerLocation(RunningContainer container)
+        internal static RunnerLocation GetRunnerLocation()
         {
-            if (knownLocation != null) return knownLocation.Value;
-            knownLocation = PingForLocation(container);
-            return knownLocation.Value;
+            DetermineRunnerLocation();
+            if (location == RunnerLocation.Unknown) throw new Exception("Runner location is unknown.");
+            return location;
         }
 
-        private static RunnerLocation PingForLocation(RunningContainer container)
+        private static void DetermineRunnerLocation()
         {
-            if (PingHost(container.Pod.PodInfo.Ip))
+            if (location != RunnerLocation.Unknown) return;
+
+            var port = Environment.GetEnvironmentVariable("KUBERNETES_PORT");
+            var host = Environment.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST");
+
+            if (string.IsNullOrEmpty(port) || string.IsNullOrEmpty(host))
             {
-                return RunnerLocation.InternalToCluster;
+                location = RunnerLocation.ExternalToCluster;
             }
-
-            foreach (var port in container.ContainerPorts)
+            else
             {
-                if (port.ExternalAddress.IsValid() && PingHost(Format(port.ExternalAddress)))
-                {
-                    return RunnerLocation.ExternalToCluster;
-                }
+                location = RunnerLocation.InternalToCluster;
             }
-
-            throw new Exception("Unable to determine location relative to kubernetes cluster.");
-        }
-
-        private static string Format(Address host)
-        {
-            return host.Host
-                .Replace("http://", "")
-                .Replace("https://", "");
-        }
-
-        private static bool PingHost(string host)
-        {
-            try
-            {
-                using var pinger = new Ping();
-                PingReply reply = pinger.Send(host);
-                return reply.Status == IPStatus.Success;
-            }
-            catch (PingException)
-            {
-            }
-
-            return false;
         }
     }
 }

Framework/KubernetesWorkflow/RunningContainers.cs

@@ -1,84 +0,0 @@
-using Utils;
-
-namespace KubernetesWorkflow
-{
-    public class RunningContainers
-    {
-        public RunningContainers(StartupConfig startupConfig, RunningPod runningPod, RunningContainer[] containers)
-        {
-            StartupConfig = startupConfig;
-            RunningPod = runningPod;
-            Containers = containers;
-        }
-
-        public StartupConfig StartupConfig { get; }
-        public RunningPod RunningPod { get; }
-        public RunningContainer[] Containers { get; }
-
-        public string Describe()
-        {
-            return string.Join(",", Containers.Select(c => c.Name));
-        }
-    }
-
-    public class RunningContainer
-    {
-        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, ContainerPort[] containerPorts)
-        {
-            Pod = pod;
-            Recipe = recipe;
-            ServicePorts = servicePorts;
-            Name = name;
-            ContainerPorts = containerPorts;
-        }
-
-        public string Name { get; }
-        public RunningPod Pod { get; }
-        public ContainerRecipe Recipe { get; }
-        public Port[] ServicePorts { get; }
-        public ContainerPort[] ContainerPorts { get; }
-
-        public ContainerPort GetContainerPort(string portTag)
-        {
-            return ContainerPorts.Single(c => c.Port.Tag == portTag);
-        }
-
-        public Address GetAddress(string portTag)
-        {
-            var containerPort = GetContainerPort(portTag);
-            if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster)
-            {
-                return containerPort.InternalAddress;
-            }
-            if (!containerPort.ExternalAddress.IsValid()) throw new Exception($"Getting address by tag {portTag} resulted in an invalid address.");
-            return containerPort.ExternalAddress;
-        }
-    }
-
-    public class ContainerPort
-    {
-        public ContainerPort(Port port, Address externalAddress, Address internalAddress)
-        {
-            Port = port;
-            ExternalAddress = externalAddress;
-            InternalAddress = internalAddress;
-        }
-
-        public Port Port { get; }
-        public Address ExternalAddress { get; }
-        public Address InternalAddress { get; }
-    }
-
-    public static class RunningContainersExtensions
-    {
-        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
-        {
-            return runningContainers.SelectMany(c => c.Containers).ToArray();
-        }
-
-        public static string Describe(this RunningContainers[] runningContainers)
-        {
-            return string.Join(",", runningContainers.Select(c => c.Describe()));
-        }
-    }
-}

Framework/KubernetesWorkflow/RunningPod.cs

@@ -1,54 +1,69 @@
-namespace KubernetesWorkflow
+using k8s;
+using k8s.Models;
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
+using Newtonsoft.Json;
+
+namespace KubernetesWorkflow
 {
-    public class RunningPod
+    public class StartResult
     {
-        public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName,  ContainerRecipePortMapEntry[] portMapEntries)
+        public StartResult(
+            K8sCluster cluster,
+            ContainerRecipe[] containerRecipes, 
+            RunningDeployment deployment,
+            RunningService? internalService,
+            RunningService? externalService)
         {
             Cluster = cluster;
-            PodInfo = podInfo;
-            DeploymentName = deploymentName;
-            ServiceName = serviceName;
-            PortMapEntries = portMapEntries;
+            ContainerRecipes = containerRecipes;
+            Deployment = deployment;
+            InternalService = internalService;
+            ExternalService = externalService;
         }
 
         public K8sCluster Cluster { get; }
-        public PodInfo PodInfo { get; }
-        public ContainerRecipePortMapEntry[] PortMapEntries { get; }
-        public string DeploymentName { get; }
-        public string ServiceName { get; }
+        public ContainerRecipe[] ContainerRecipes { get; }
+        public RunningDeployment Deployment { get; }
+        public RunningService? InternalService { get; }
+        public RunningService? ExternalService { get; }
 
-        public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
+        public Port GetInternalServicePorts(ContainerRecipe recipe, string tag)
         {
-            return PortMapEntries
-                .Where(p => p.ContainerNumber == containerRecipe.Number)
-                .SelectMany(p => p.Ports)
-                .ToArray();
-        }
-    }
+            if (InternalService != null)
+            {
+                var p = InternalService.GetServicePortForRecipeAndTag(recipe, tag);
+                if (p != null) return p;
+            }
 
-    public class ContainerRecipePortMapEntry
-    {
-        public ContainerRecipePortMapEntry(int containerNumber, Port[] ports)
-        {
-            ContainerNumber = containerNumber;
-            Ports = ports;
+            throw new Exception($"Unable to find internal port by tag '{tag}' for recipe '{recipe.Name}'.");
         }
 
-        public int ContainerNumber { get; }
-        public Port[] Ports { get; }
-    }
-
-    public class PodInfo
-    {
-        public PodInfo(string name, string ip, string k8sNodeName)
+        public Port GetExternalServicePorts(ContainerRecipe recipe, string tag)
         {
-            Name = name;
-            Ip = ip;
-            K8SNodeName = k8sNodeName;
+            if (ExternalService != null)
+            {
+                var p = ExternalService.GetServicePortForRecipeAndTag(recipe, tag);
+                if (p != null) return p;
+            }
+
+            throw new Exception($"Unable to find external port by tag '{tag}' for recipe '{recipe.Name}'.");
         }
 
-        public string Name { get; }
-        public string Ip { get; }
-        public string K8SNodeName { get; }
+        public Port[] GetServicePortsForContainer(ContainerRecipe recipe)
+        {
+            if (InternalService != null)
+            {
+                var p = InternalService.GetServicePortsForRecipe(recipe);
+                if (p.Any()) return p;
+            }
+            if (ExternalService != null)
+            {
+                var p = ExternalService.GetServicePortsForRecipe(recipe);
+                if (p.Any()) return p;
+            }
+
+            return Array.Empty<Port>();
+        }
     }
 }

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -1,4 +1,6 @@
-using Logging;
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
+using Logging;
 using Newtonsoft.Json;
 using Utils;
 
@@ -9,8 +11,10 @@ namespace KubernetesWorkflow
         IKnownLocations GetAvailableLocations();
         RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
         RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        PodInfo GetPodInfo(RunningContainer container);
+        PodInfo GetPodInfo(RunningContainers containers);
         CrashWatcher CreateCrashWatcher(RunningContainer container);
-        void Stop(RunningContainers runningContainers);
+        void Stop(RunningContainers containers, bool waitTillStopped);
         void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
         void DeleteNamespace();
@@ -50,26 +54,43 @@ namespace KubernetesWorkflow
         {
             return K8s(controller =>
             {
-                var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
-                var runningPod = controller.BringOnline(recipes, location);
-                var containers = CreateContainers(runningPod, recipes, startupConfig);
+                componentFactory.Update(controller);
 
-                var rc = new RunningContainers(startupConfig, runningPod, containers);
+                var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
+                var startResult = controller.BringOnline(recipes, location);
+                var containers = CreateContainers(startResult, recipes, startupConfig);
+
+                var rc = new RunningContainers(startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
+
+                if (startResult.ExternalService != null)
+                {
+                    componentFactory.Update(controller);
+                }
                 return rc;
             });
         }
 
+        public PodInfo GetPodInfo(RunningContainer container)
+        {
+            return K8s(c => c.GetPodInfo(container.RunningContainers.StartResult.Deployment));
+        }
+
+        public PodInfo GetPodInfo(RunningContainers containers)
+        {
+            return K8s(c => c.GetPodInfo(containers.StartResult.Deployment));
+        }
+
         public CrashWatcher CreateCrashWatcher(RunningContainer container)
         {
             return K8s(c => c.CreateCrashWatcher(container));
         }
 
-        public void Stop(RunningContainers runningContainers)
+        public void Stop(RunningContainers runningContainers, bool waitTillStopped)
         {
             K8s(controller =>
             {
-                controller.Stop(runningContainers.RunningPod);
+                controller.Stop(runningContainers.StartResult, waitTillStopped);
                 cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
             });
         }
@@ -78,7 +99,7 @@ namespace KubernetesWorkflow
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler, tailLines);
+                controller.DownloadPodLog(container, logHandler, tailLines);
             });
         }
 
@@ -86,7 +107,7 @@ namespace KubernetesWorkflow
         {
             return K8s(controller =>
             {
-                return controller.ExecuteCommand(container.Pod, container.Recipe.Name, command, args);
+                return controller.ExecuteCommand(container, command, args);
             });
         }
 
@@ -106,18 +127,16 @@ namespace KubernetesWorkflow
             });
         }
 
-        private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig)
+        private RunningContainer[] CreateContainers(StartResult startResult, ContainerRecipe[] recipes, StartupConfig startupConfig)
         {
             log.Debug();
             return recipes.Select(r =>
             {
-                var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
-                log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
-
                 var name = GetContainerName(r, startupConfig);
+                var addresses = CreateContainerAddresses(startResult, r);
+                log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}");
 
-                return new RunningContainer(runningPod, r, servicePorts, name,
-                    CreateContainerPorts(runningPod, r, servicePorts));
+                return new RunningContainer(name, r, addresses);
 
             }).ToArray();
         }
@@ -135,43 +154,39 @@ namespace KubernetesWorkflow
             }
         }
 
-        private ContainerPort[] CreateContainerPorts(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts)
+        private ContainerAddress[] CreateContainerAddresses(StartResult startResult, ContainerRecipe recipe)
         {
-            var result = new List<ContainerPort>();
+            var result = new List<ContainerAddress>();
             foreach (var exposedPort in recipe.ExposedPorts)
             {
-                result.Add(new ContainerPort(
-                    exposedPort,
-                    GetContainerExternalAddress(pod, servicePorts, exposedPort),
-                    GetContainerInternalAddress(pod, exposedPort)));
+                result.Add(new ContainerAddress(exposedPort.Tag, GetContainerExternalAddress(startResult, recipe, exposedPort.Tag), false));
+                result.Add(new ContainerAddress(exposedPort.Tag, GetContainerInternalAddress(startResult, recipe, exposedPort.Tag), true));
             }
             foreach (var internalPort in recipe.InternalPorts)
             {
-                if (!string.IsNullOrEmpty(internalPort.Tag))
-                {
-                    result.Add(new ContainerPort(
-                        internalPort,
-                        new Address(string.Empty, 0),
-                        GetContainerInternalAddress(pod, internalPort)));
-                }
+                result.Add(new ContainerAddress(internalPort.Tag, GetContainerInternalAddress(startResult, recipe, internalPort.Tag), true));
             }
 
             return result.ToArray();
         }
 
-        private static Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts, Port exposedPort)
+        private static Address GetContainerExternalAddress(StartResult startResult, ContainerRecipe recipe, string tag)
         {
-            var servicePort = servicePorts.Single(p => p.Tag == exposedPort.Tag);
+            var port = startResult.GetExternalServicePorts(recipe, tag);
 
             return new Address(
-                pod.Cluster.HostAddress,
-                servicePort.Number);
+                startResult.Cluster.HostAddress,
+                port.Number);
         }
 
-        private Address GetContainerInternalAddress(RunningPod pod, Port port)
+        private Address GetContainerInternalAddress(StartResult startResult, ContainerRecipe recipe, string tag)
         {
+            var namespaceName = startResult.Cluster.Configuration.KubernetesNamespace;
+            var serviceName = startResult.InternalService!.Name;
+            var port = startResult.GetInternalServicePorts(recipe, tag);
+
             return new Address(
-                $"http://{pod.PodInfo.Ip}",
+                $"http://{serviceName}.{namespaceName}.svc.cluster.local",
                 port.Number);
         }
         
@@ -182,6 +197,8 @@ namespace KubernetesWorkflow
             for (var i = 0; i < numberOfContainers; i++)
             {
                 var recipe = recipeFactory.CreateRecipe(i, numberSource.GetContainerNumber(), componentFactory, startupConfig);
+                CheckPorts(recipe);
+
                 if (cluster.Configuration.AddAppPodLabel) recipe.PodLabels.Add("app", recipeFactory.AppName);
                 cluster.Configuration.Hooks.OnContainerRecipeCreated(recipe);
                 result.Add(recipe);
@@ -190,6 +207,18 @@ namespace KubernetesWorkflow
             return result.ToArray();
         }
 
+        private void CheckPorts(ContainerRecipe recipe)
+        {
+            var allTags =
+                recipe.ExposedPorts.Concat(recipe.InternalPorts)
+                .Select(p => K8sNameUtils.Format(p.Tag)).ToArray();
+
+            if (allTags.Length != allTags.Distinct().Count())
+            {
+                throw new Exception("Duplicate port tags found in recipe for " + recipe.Name);
+            }
+        }
+
         private void K8s(Action<K8sController> action)
         {
             try
@@ -200,7 +229,7 @@ namespace KubernetesWorkflow
             }
             catch (k8s.Autorest.HttpOperationException ex)
             {
-                log.Error(JsonConvert.SerializeObject(ex));
+                log.Error(JsonConvert.SerializeObject(ex.Response));
                 throw;
             }
         }
@@ -216,30 +245,9 @@ namespace KubernetesWorkflow
             }
             catch (k8s.Autorest.HttpOperationException ex)
             {
-                log.Error(JsonConvert.SerializeObject(ex));
+                log.Error(JsonConvert.SerializeObject(ex.Response));
                 throw;
             }
         }
     }
-
-    public interface ILogHandler
-    {
-        void Log(Stream log);
-    }
-
-    public abstract class LogHandler : ILogHandler
-    {
-        public void Log(Stream log)
-        {
-            using var reader = new StreamReader(log);
-            var line = reader.ReadLine();
-            while (line != null)
-            {
-                ProcessLine(line);
-                line = reader.ReadLine();
-            }
-        }
-
-        protected abstract void ProcessLine(string line);
-    }
 }

Framework/KubernetesWorkflow/Types/ContainerAddress.cs

@@ -0,0 +1,24 @@
+using Utils;
+
+namespace KubernetesWorkflow.Types
+{
+    public class ContainerAddress
+    {
+        public ContainerAddress(string portTag, Address address, bool isInteral)
+        {
+            PortTag = portTag;
+            Address = address;
+            IsInteral = isInteral;
+        }
+
+        public string PortTag { get; }
+        public Address Address { get; }
+        public bool IsInteral { get; }
+
+        public override string ToString()
+        {
+            var indicator = IsInteral ? "int" : "ext";
+            return $"{indicator} {PortTag} -> '{Address}'";
+        }
+    }
+}

Framework/KubernetesWorkflow/Types/ContainerRecipePortMapEntry.cs

@@ -0,0 +1,16 @@
+using KubernetesWorkflow.Recipe;
+
+namespace KubernetesWorkflow.Types
+{
+    public class ContainerRecipePortMapEntry
+    {
+        public ContainerRecipePortMapEntry(int recipeNumber, Port[] ports)
+        {
+            RecipeNumber = recipeNumber;
+            Ports = ports;
+        }
+
+        public int RecipeNumber { get; }
+        public Port[] Ports { get; }
+    }
+}

Framework/KubernetesWorkflow/Types/K8sNodeLabel.cs

@@ -0,0 +1,14 @@
+namespace KubernetesWorkflow.Types
+{
+    public class K8sNodeLabel
+    {
+        public K8sNodeLabel(string key, string value)
+        {
+            Key = key;
+            Value = value;
+        }
+
+        public string Key { get; }
+        public string Value { get; }
+    }
+}

Framework/KubernetesWorkflow/Types/PodInfo.cs

@@ -0,0 +1,16 @@
+namespace KubernetesWorkflow.Types
+{
+    public class PodInfo
+    {
+        public PodInfo(string name, string ip, string k8sNodeName)
+        {
+            Name = name;
+            Ip = ip;
+            K8SNodeName = k8sNodeName;
+        }
+
+        public string Name { get; }
+        public string Ip { get; }
+        public string K8SNodeName { get; }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -0,0 +1,54 @@
+using KubernetesWorkflow.Recipe;
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+
+namespace KubernetesWorkflow.Types
+{
+    public class RunningContainer
+    {
+        public RunningContainer(string name, ContainerRecipe recipe, ContainerAddress[] addresses)
+        {
+            Name = name;
+            Recipe = recipe;
+            Addresses = addresses;
+        }
+
+        public string Name { get; }
+        public ContainerRecipe Recipe { get; }
+        public ContainerAddress[] Addresses { get; }
+
+        [JsonIgnore]
+        public RunningContainers RunningContainers { get; internal set; } = null!;
+
+        public Address GetAddress(ILog log, string portTag)
+        {
+            var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
+            if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);
+
+            var select = SelectAddress(addresses);
+            log.Debug($"Container '{Name}' selected for tag '{portTag}' address: '{select}'");
+            return select.Address;
+        }
+
+        public Address GetInternalAddress(string portTag)
+        {
+            var containerAddress = Addresses.Single(a => a.PortTag == portTag && a.IsInteral);
+            return containerAddress.Address;
+        }
+
+        private ContainerAddress SelectAddress(ContainerAddress[] addresses)
+        {
+            var location = RunnerLocationUtils.GetRunnerLocation();
+            if (location == RunnerLocation.InternalToCluster)
+            {
+                return addresses.Single(a => a.IsInteral);
+            }
+            if (location == RunnerLocation.ExternalToCluster)
+            {
+                return addresses.Single(a => !a.IsInteral);
+            }
+            throw new Exception("Running location not known.");
+        }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningContainers.cs

@@ -0,0 +1,44 @@
+using Newtonsoft.Json;
+
+namespace KubernetesWorkflow.Types
+{
+    public class RunningContainers
+    {
+        public RunningContainers(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        {
+            StartupConfig = startupConfig;
+            StartResult = startResult;
+            Containers = containers;
+
+            foreach (var c in containers) c.RunningContainers = this;
+        }
+
+        public StartupConfig StartupConfig { get; }
+        public StartResult StartResult { get; }
+        public RunningContainer[] Containers { get; }
+
+        [JsonIgnore]
+        public string Name
+        {
+            get { return $"{Containers.Length}x '{Containers.First().Name}'"; }
+        }
+
+        public string Describe()
+        {
+            return string.Join(",", Containers.Select(c => c.Name));
+        }
+    }
+
+    public static class RunningContainersExtensions
+    {
+        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
+        {
+            return runningContainers.SelectMany(c => c.Containers).ToArray();
+        }
+
+        public static string Describe(this RunningContainers[] runningContainers)
+        {
+            return string.Join(",", runningContainers.Select(c => c.Describe()));
+        }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningDeployment.cs

@@ -0,0 +1,14 @@
+namespace KubernetesWorkflow.Types
+{
+    public class RunningDeployment
+    {
+        public RunningDeployment(string name, string podLabel)
+        {
+            Name = name;
+            PodLabel = podLabel;
+        }
+
+        public string Name { get; }
+        public string PodLabel { get; }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningService.cs

@@ -0,0 +1,29 @@
+using KubernetesWorkflow.Recipe;
+
+namespace KubernetesWorkflow.Types
+{
+    public class RunningService
+    {
+        public RunningService(string name, List<ContainerRecipePortMapEntry> result)
+        {
+            Name = name;
+            Result = result;
+        }
+
+        public string Name { get; }
+        public List<ContainerRecipePortMapEntry> Result { get; }
+
+        public Port? GetServicePortForRecipeAndTag(ContainerRecipe recipe, string tag)
+        {
+            return GetServicePortsForRecipe(recipe).SingleOrDefault(p => p.Tag == tag);
+        }
+
+        public Port[] GetServicePortsForRecipe(ContainerRecipe recipe)
+        {
+            return Result
+                .Where(p => p.RecipeNumber == recipe.Number)
+                .SelectMany(p => p.Ports)
+                .ToArray();
+        }
+    }
+}

Framework/Logging/BaseLog.cs

@@ -13,16 +13,21 @@ namespace Logging
 
     public abstract class BaseLog : ILog
     {
+        public static bool EnableDebugLogging { get; set; } = false;
+
         private readonly NumberSource subfileNumberSource = new NumberSource(0);
-        private readonly bool debug;
         private readonly List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
         private LogFile? logFile;
 
-        protected BaseLog(bool debug)
+        public BaseLog()
         {
-            this.debug = debug;
+            IsDebug =
+                EnableDebugLogging ||
+                !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("LOGDEBUG")) ||
+                !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("DEBUGLOG"));
         }
 
+        protected bool IsDebug { get; private set; }
         protected abstract string GetFullName();
 
         public LogFile LogFile 
@@ -41,7 +46,7 @@ namespace Logging
 
         public void Debug(string message = "", int skipFrames = 0)
         {
-            if (debug)
+            if (IsDebug)
             {
                 var callerName = DebugStack.GetCallerName(skipFrames);
                 Log($"(debug)({callerName}) {message}");
@@ -50,7 +55,9 @@ namespace Logging
 
         public virtual void Error(string message)
         {
-            Log($"[ERROR] {message}");
+            var msg = $"[ERROR] {message}";
+            Console.WriteLine(msg);
+            Log(msg);
         }
 
         public virtual void AddStringReplace(string from, string to)
@@ -72,7 +79,7 @@ namespace Logging
 
         private string ApplyReplacements(string str)
         {
-            if (debug) return str;
+            if (IsDebug) return str;
             foreach (var replacement in replacements)
             {
                 str = replacement.Apply(str);

Framework/Logging/ConsoleLog.cs

@@ -2,10 +2,6 @@
 {
     public class ConsoleLog : BaseLog
     {
-        public ConsoleLog() : base(false)
-        {
-        }
-
         protected override string GetFullName()
         {
             return "CONSOLE";

Framework/Logging/FileLog.cs

@@ -0,0 +1,17 @@
+namespace Logging
+{
+    public class FileLog : BaseLog
+    {
+        public FileLog(string fullFilename)
+        {
+            FullFilename = fullFilename;
+        }
+
+        public string FullFilename { get; }
+
+        protected override string GetFullName()
+        {
+            return FullFilename;
+        }
+    }
+}

Framework/Logging/LogConfig.cs

@@ -2,13 +2,11 @@
 {
     public class LogConfig
     {
-        public LogConfig(string logRoot, bool debugEnabled)
+        public LogConfig(string logRoot)
         {
             LogRoot = logRoot;
-            DebugEnabled = debugEnabled;
         }
         
         public string LogRoot { get; }
-        public bool DebugEnabled { get; }
     }
 }

Framework/Logging/LogPrefixer.cs

@@ -3,14 +3,21 @@
     public class LogPrefixer : ILog
     {
         private readonly ILog backingLog;
-        private readonly string prefix;
+
+        public LogPrefixer(ILog backingLog)
+        {
+            this.backingLog = backingLog;
+        }
 
         public LogPrefixer(ILog backingLog, string prefix)
         {
             this.backingLog = backingLog;
-            this.prefix = prefix;
+            Prefix = prefix;
         }
 
+        public string Prefix { get; set; } = string.Empty;
+
+
         public LogFile CreateSubfile(string ext = "log")
         {
             return backingLog.CreateSubfile(ext);
@@ -18,17 +25,17 @@
 
         public void Debug(string message = "", int skipFrames = 0)
         {
-            backingLog.Debug(prefix + message, skipFrames);
+            backingLog.Debug(Prefix + message, skipFrames);
         }
 
         public void Error(string message)
         {
-            backingLog.Error(prefix + message);
+            backingLog.Error(Prefix + message);
         }
 
         public void Log(string message)
         {
-            backingLog.Log(prefix + message);
+            backingLog.Log(Prefix + message);
         }
 
         public void AddStringReplace(string from, string to)

Framework/Logging/NullLog.cs

@@ -2,10 +2,6 @@
 {
     public class NullLog : BaseLog
     {
-        public NullLog() : base(false)
-        {
-        }
-
         public string FullFilename { get; set; } = "NULL";
 
         protected override string GetFullName()
@@ -15,11 +11,13 @@
 
         public override void Log(string message)
         {
+            if (IsDebug) base.Log(message);
         }
 
         public override void Error(string message)
         {
             Console.WriteLine("Error: " + message);
+            base.Error(message);
         }
 
         public override void AddStringReplace(string from, string to)

Framework/Logging/Stopwatch.cs

@@ -16,19 +16,19 @@ namespace Logging
             this.debug = debug;
         }
 
-        public static void Measure(ILog log, string name, Action action, bool debug = false)
+        public static TimeSpan Measure(ILog log, string name, Action action, bool debug = false)
         {
             var sw = Begin(log, name, debug);
             action();
-            sw.End();
+            return sw.End();
         }
 
-        public static T Measure<T>(ILog log, string name, Func<T> action, bool debug = false)
+        public static StopwatchResult<T> Measure<T>(ILog log, string name, Func<T> action, bool debug = false)
         {
             var sw = Begin(log, name, debug);
             var result = action();
-            sw.End();
-            return result;
+            var duration = sw.End();
+            return new StopwatchResult<T>(result, duration);
         }
 
         public static Stopwatch Begin(ILog log)
@@ -68,4 +68,16 @@ namespace Logging
             return duration;
         }
     }
+
+    public class StopwatchResult<T>
+    {
+        public StopwatchResult(T value, TimeSpan duration)
+        {
+            Value = value;
+            Duration = duration;
+        }
+
+        public T Value { get; }
+        public TimeSpan Duration { get; }
+    }
 }

Framework/NethereumWorkflow/BlockUtils/BlockCache.cs

@@ -0,0 +1,39 @@
+namespace NethereumWorkflow.BlockUtils
+{
+    public class BlockCache
+    {
+        public delegate void CacheClearedEvent();
+
+        private const int MaxEntries = 1024 * 1024 * 5;
+        private readonly Dictionary<ulong, BlockTimeEntry> entries = new Dictionary<ulong, BlockTimeEntry>();
+
+        public event CacheClearedEvent? OnCacheCleared;
+
+        public BlockTimeEntry Add(ulong number, DateTime dateTime)
+        {
+            return Add(new BlockTimeEntry(number, dateTime));
+        }
+
+        public BlockTimeEntry Add(BlockTimeEntry entry)
+        {
+            if (!entries.ContainsKey(entry.BlockNumber))
+            {
+                if (entries.Count > MaxEntries)
+                {
+                    entries.Clear();
+                    var e = OnCacheCleared;
+                    if (e != null) e();
+                }
+                entries.Add(entry.BlockNumber, entry);
+            }
+
+            return entries[entry.BlockNumber];
+        }
+
+        public BlockTimeEntry? Get(ulong number)
+        {
+            if (!entries.TryGetValue(number, out BlockTimeEntry? value)) return null;
+            return value;
+        }
+    }
+}

Framework/NethereumWorkflow/BlockUtils/BlockTimeEntry.cs

@@ -0,0 +1,19 @@
+namespace NethereumWorkflow.BlockUtils
+{
+    public class BlockTimeEntry
+    {
+        public BlockTimeEntry(ulong blockNumber, DateTime utc)
+        {
+            BlockNumber = blockNumber;
+            Utc = utc;
+        }
+
+        public ulong BlockNumber { get; }
+        public DateTime Utc { get; }
+
+        public override string ToString()
+        {
+            return $"[{BlockNumber}] @ {Utc.ToString("o")}";
+        }
+    }
+}

Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs

@@ -0,0 +1,95 @@
+using Logging;
+
+namespace NethereumWorkflow.BlockUtils
+{
+    public class BlockTimeFinder
+    {
+        private readonly BlockCache cache;
+        private readonly BlockchainBounds bounds;
+        private readonly IWeb3Blocks web3;
+        private readonly ILog log;
+
+        public BlockTimeFinder(BlockCache cache, IWeb3Blocks web3, ILog log)
+        {
+            this.web3 = web3;
+            this.log = log;
+
+            this.cache = cache;
+            bounds = new BlockchainBounds(cache, web3);
+        }
+
+        public ulong? GetHighestBlockNumberBefore(DateTime moment)
+        {
+            bounds.Initialize();
+            if (moment <= bounds.Genesis.Utc) return null;
+            if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
+
+            return Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector);
+        }
+
+        public ulong? GetLowestBlockNumberAfter(DateTime moment)
+        {
+            bounds.Initialize();
+            if (moment >= bounds.Current.Utc) return null;
+            if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
+
+            return Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector);
+        }
+
+        private ulong Search(BlockTimeEntry lower, BlockTimeEntry upper, DateTime target, Func<DateTime, BlockTimeEntry, bool> isWhatIwant)
+        {
+            var middle = GetMiddle(lower, upper);
+            if (middle.BlockNumber == lower.BlockNumber)
+            {
+                if (isWhatIwant(target, upper)) return upper.BlockNumber;
+            }
+
+            if (isWhatIwant(target, middle))
+            {
+                return middle.BlockNumber;
+            }
+
+            if (middle.Utc > target)
+            {
+                return Search(lower, middle, target, isWhatIwant);
+            }
+            else
+            {
+                return Search(middle, upper, target, isWhatIwant);
+            }
+        }
+
+        private BlockTimeEntry GetMiddle(BlockTimeEntry lower, BlockTimeEntry upper)
+        {
+            ulong range = upper.BlockNumber - lower.BlockNumber;
+            ulong number = lower.BlockNumber + range / 2;
+            return GetBlock(number);
+        }
+
+        private bool HighestBeforeSelector(DateTime target, BlockTimeEntry entry)
+        {
+            var next = GetBlock(entry.BlockNumber + 1);
+            return
+                entry.Utc < target &&
+                next.Utc > target;
+        }
+
+        private bool LowestAfterSelector(DateTime target, BlockTimeEntry entry)
+        {
+            var previous = GetBlock(entry.BlockNumber - 1);
+            return
+                entry.Utc > target &&
+                previous.Utc < target;
+        }
+
+        private BlockTimeEntry GetBlock(ulong number)
+        {
+            if (number < bounds.Genesis.BlockNumber) throw new Exception("Can't fetch block before genesis.");
+            if (number > bounds.Current.BlockNumber) throw new Exception("Can't fetch block after current.");
+
+            var dateTime = web3.GetTimestampForBlock(number);
+            if (dateTime == null) throw new Exception("Failed to get dateTime for block that should exist.");
+            return cache.Add(number, dateTime.Value);
+        }
+    }
+}

Framework/NethereumWorkflow/BlockUtils/BlockchainBounds.cs

@@ -0,0 +1,106 @@
+namespace NethereumWorkflow.BlockUtils
+{
+    public class BlockchainBounds
+    {
+        private readonly BlockCache cache;
+        private readonly IWeb3Blocks web3;
+
+        public BlockTimeEntry Genesis { get; private set; } = null!;
+        public BlockTimeEntry Current { get; private set; } = null!;
+
+        public BlockchainBounds(BlockCache cache, IWeb3Blocks web3)
+        {
+            this.cache = cache;
+            this.web3 = web3;
+
+            cache.OnCacheCleared += Initialize;
+        }
+
+        public void Initialize()
+        {
+            AddCurrentBlock();
+            LookForGenesisBlock();
+
+            if (Current.BlockNumber == Genesis.BlockNumber)
+            {
+                throw new Exception("Unsupported condition: Current block is genesis block.");
+            }
+        }
+
+        private void LookForGenesisBlock()
+        {
+            if (Genesis != null)
+            {
+                cache.Add(Genesis);
+                return;
+            }
+
+            var blockTime = web3.GetTimestampForBlock(0);
+            if (blockTime != null)
+            {
+                AddGenesisBlock(0, blockTime.Value);
+                return;
+            }
+
+            LookForGenesisBlock(0, Current);
+        }
+
+        private void LookForGenesisBlock(ulong lower, BlockTimeEntry upper)
+        {
+            if (Genesis != null) return;
+
+            var range = upper.BlockNumber - lower;
+            if (range == 1)
+            {
+                var lowTime = web3.GetTimestampForBlock(lower);
+                if (lowTime != null)
+                {
+                    AddGenesisBlock(lower, lowTime.Value);
+                }
+                else
+                {
+                    AddGenesisBlock(upper);
+                }
+                return;
+            }
+
+            var current = lower + range / 2;
+
+            var blockTime = web3.GetTimestampForBlock(current);
+            if (blockTime != null)
+            {
+                var newUpper = cache.Add(current, blockTime.Value);
+                LookForGenesisBlock(lower, newUpper);
+            }
+            else
+            {
+                LookForGenesisBlock(current, upper);
+            }
+        }
+
+        private void AddCurrentBlock()
+        {
+            var currentBlockNumber = web3.GetCurrentBlockNumber();
+            var blockTime = web3.GetTimestampForBlock(currentBlockNumber);
+            if (blockTime == null) throw new Exception("Unable to get dateTime for current block.");
+            AddCurrentBlock(currentBlockNumber, blockTime.Value);
+        }
+
+        private void AddCurrentBlock(ulong currentBlockNumber, DateTime dateTime)
+        {
+            Current = new BlockTimeEntry(currentBlockNumber, dateTime);
+            cache.Add(Current);
+        }
+
+        private void AddGenesisBlock(ulong number, DateTime dateTime)
+        {
+            AddGenesisBlock(new BlockTimeEntry(number, dateTime));
+        }
+
+        private void AddGenesisBlock(BlockTimeEntry entry)
+        {
+            Genesis = entry;
+            cache.Add(Genesis);
+        }
+    }
+}

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -1,13 +1,18 @@
 using Logging;
+using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
+using NethereumWorkflow.BlockUtils;
 using Utils;
 
 namespace NethereumWorkflow
 {
     public class NethereumInteraction
     {
+        // BlockCache is a static instance: It stays alive for the duration of the application runtime.
+        private readonly static BlockCache blockCache = new BlockCache();
+
         private readonly ILog log;
         private readonly Web3 web3;
 
@@ -17,11 +22,12 @@ namespace NethereumWorkflow
             this.web3 = web3;
         }
 
-        public void SendEth(string toAddress, decimal ethAmount)
+        public string SendEth(string toAddress, decimal ethAmount)
         {
             log.Debug();
             var receipt = Time.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(toAddress, ethAmount));
             if (!receipt.Succeeded()) throw new Exception("Unable to send Eth");
+            return receipt.TransactionHash;
         }
 
         public decimal GetEthBalance()
@@ -44,12 +50,19 @@ namespace NethereumWorkflow
             return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function));
         }
 
-        public void SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
+        public string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
         {
             log.Debug();
             var handler = web3.Eth.GetContractTransactionHandler<TFunction>();
             var receipt = Time.Wait(handler.SendRequestAndWaitForReceiptAsync(contractAddress, function));
             if (!receipt.Succeeded()) throw new Exception("Unable to perform contract transaction.");
+            return receipt.TransactionHash;
+        }
+
+        public Transaction GetTransaction(string transactionHash)
+        {
+            log.Debug();
+            return Time.Wait(web3.Eth.Transactions.GetTransactionByHash.SendRequestAsync(transactionHash));
         }
 
         public decimal? GetSyncedBlockNumber()
@@ -75,5 +88,36 @@ namespace NethereumWorkflow
                 return false;
             }
         }
+
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new()
+        {
+            var wrapper = new Web3Wrapper(web3, log);
+            var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log);
+
+            var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From);
+            var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To);
+
+            if (!fromBlock.HasValue)
+            {
+                log.Error("Failed to find lowest block for time range: " + timeRange);
+                throw new Exception("Failed");
+            }
+            if (!toBlock.HasValue)
+            {
+                log.Error("Failed to find highest block for time range: " + timeRange);
+                throw new Exception("Failed");
+            }
+
+            return GetEvents<TEvent>(address, fromBlock.Value, toBlock.Value);
+        }
+
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new()
+        {
+            var eventHandler = web3.Eth.GetEvent<TEvent>(address);
+            var from = new BlockParameter(fromBlockNumber);
+            var to = new BlockParameter(toBlockNumber);
+            var blockFilter = Time.Wait(eventHandler.CreateFilterBlockRangeAsync(from, to));
+            return Time.Wait(eventHandler.GetAllChangesAsync(blockFilter));
+        }
     }
 }

Framework/NethereumWorkflow/Web3Wrapper.cs

@@ -0,0 +1,46 @@
+using Logging;
+using Nethereum.RPC.Eth.DTOs;
+using Nethereum.Web3;
+using Utils;
+
+namespace NethereumWorkflow
+{
+    public interface IWeb3Blocks
+    {
+        ulong GetCurrentBlockNumber();
+        DateTime? GetTimestampForBlock(ulong blockNumber);
+    }
+
+    public class Web3Wrapper : IWeb3Blocks
+    {
+        private readonly Web3 web3;
+        private readonly ILog log;
+
+        public Web3Wrapper(Web3 web3, ILog log)
+        {
+            this.web3 = web3;
+            this.log = log;
+        }
+
+        public ulong GetCurrentBlockNumber()
+        {
+            var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
+            return Convert.ToUInt64(number.ToDecimal());
+        }
+
+        public DateTime? GetTimestampForBlock(ulong blockNumber)
+        {
+            try
+            {
+                var block = Time.Wait(web3.Eth.Blocks.GetBlockWithTransactionsByNumber.SendRequestAsync(new BlockParameter(blockNumber)));
+                if (block == null) return null;
+                return DateTimeOffset.FromUnixTimeSeconds(Convert.ToInt64(block.Timestamp.ToDecimal())).UtcDateTime;
+            }
+            catch (Exception ex)
+            {
+                log.Error("Exception while getting timestamp for block: " + ex);
+                return null;
+            }
+        }
+    }
+}

Framework/Utils/ByteSize.cs

@@ -2,6 +2,9 @@
 {
     public class ByteSize
     {
+        public static readonly ByteSize Zero = new ByteSize(0);
+        public const double DefaultSecondsPerMB = 10.0;
+
         public ByteSize(long sizeInBytes)
         {
             if (sizeInBytes < 0) throw new ArgumentException("Cannot create ByteSize object with size less than 0. Was: " + sizeInBytes);
@@ -10,7 +13,6 @@
 
         public long SizeInBytes { get; }
 
-        public const double DefaultSecondsPerMB = 10.0;
 
         public long ToMB()
         {
@@ -46,6 +48,18 @@
         }
     }
 
+    public class BytesPerSecond : ByteSize
+    {
+        public BytesPerSecond(long sizeInBytes) : base(sizeInBytes)
+        {
+        }
+
+        public override string ToString()
+        {
+            return base.ToString() + "/s";
+        }
+    }
+
     public static class ByteSizeIntExtensions
     {
         private const long Kilo = 1024;

Framework/Utils/TaskFactory.cs

@@ -1,15 +1,27 @@
-namespace ContinuousTests
+namespace Utils
 {
     public class TaskFactory
     {
         private readonly object taskLock = new();
         private readonly List<Task> activeTasks = new List<Task>();
 
-        public void Run(Action action)
+        public void Run(Action action, string name)
         {
             lock (taskLock)
             {
-                activeTasks.Add(Task.Run(action).ContinueWith(CleanupTask, null));
+                activeTasks.Add(Task.Run(() => CatchException(action, name)).ContinueWith(CleanupTask, null));
+            }
+        }
+
+        private void CatchException(Action action, string name)
+        {
+            try
+            {
+                action();
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Exception in task '{name}': " + ex);
             }
         }
 

Framework/Utils/Time.cs

@@ -13,6 +13,11 @@
             return task.Result;
         }
 
+        public static void Wait(Task task)
+        {
+            task.Wait();
+        }
+
         public static string FormatDuration(TimeSpan d)
         {
             var result = "";
@@ -23,6 +28,35 @@
             return result;
         }
         
+        public static TimeSpan ParseTimespan(string span)
+        {
+            span = span.Replace(" ", "").Replace(",", "");
+            var result = TimeSpan.Zero;
+            var number = "";
+            foreach (var c in span)
+            {
+                if (char.IsNumber(c)) number += c;
+                else
+                {
+                    var value = Convert.ToInt32(number);
+                    number = "";
+
+                    if (c == 'd') result += TimeSpan.FromDays(value);
+                    else if (c == 'h') result += TimeSpan.FromHours(value);
+                    else if (c == 'm') result += TimeSpan.FromMinutes(value);
+                    else if (c == 's') result += TimeSpan.FromSeconds(value);
+                    else throw new Exception("Unknown time modifier: " + c);
+                }
+            }
+
+            if (!string.IsNullOrEmpty(number))
+            {
+                var value = Convert.ToInt32(number);
+                result += TimeSpan.FromSeconds(value);
+            }
+            return result;
+        }
+
         public static void WaitUntil(Func<bool> predicate)
         {
             WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
@@ -56,12 +90,12 @@
 
         public static void Retry(Action action, int maxRetries, string description)
         {
-            Retry(action, maxRetries, TimeSpan.FromSeconds(1), description);
+            Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
         }
 
         public static T Retry<T>(Func<T> action, int maxRetries, string description)
         {
-            return Retry(action, maxRetries, TimeSpan.FromSeconds(1), description);
+            return Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
         }
 
         public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)

Framework/Utils/TimeRange.cs

@@ -0,0 +1,24 @@
+namespace Utils
+{
+    public class TimeRange
+    {
+        public TimeRange(DateTime from, DateTime to)
+        {
+            if (from < to)
+            {
+                From = from;
+                To = to;
+            }
+            else
+            {
+                From = to;
+                To = from;
+            }
+            Duration = To - From;
+        }
+
+        public DateTime From { get; }
+        public DateTime To { get; }
+        public TimeSpan Duration { get; }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -1,5 +1,11 @@
-using GethPlugin;
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
 using Logging;
+using Nethereum.ABI;
+using Nethereum.Hex.HexTypes;
+using Nethereum.Util;
+using NethereumWorkflow;
+using Utils;
 
 namespace CodexContractsPlugin
 {
@@ -7,10 +13,28 @@ namespace CodexContractsPlugin
     {
         CodexContractsDeployment Deployment { get; }
 
-        void MintTestTokens(IHasEthAddress owner, TestToken testTokens);
-        void MintTestTokens(EthAddress ethAddress, TestToken testTokens);
+        bool IsDeployed();
+        string MintTestTokens(IHasEthAddress owner, TestToken testTokens);
+        string MintTestTokens(EthAddress ethAddress, TestToken testTokens);
         TestToken GetTestTokenBalance(IHasEthAddress owner);
         TestToken GetTestTokenBalance(EthAddress ethAddress);
+
+        Request[] GetStorageRequests(TimeRange timeRange);
+        EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
+        RequestState GetRequestState(Request request);
+        RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange);
+        RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange);
+        SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange);
+        SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange);
+    }
+
+    public enum RequestState
+    {
+        New,
+        Started,
+        Cancelled,
+        Finished,
+        Failed
     }
 
     public class CodexContractsAccess : ICodexContracts
@@ -27,15 +51,19 @@ namespace CodexContractsPlugin
 
         public CodexContractsDeployment Deployment { get; }
 
-        public void MintTestTokens(IHasEthAddress owner, TestToken testTokens)
+        public bool IsDeployed()
         {
-            MintTestTokens(owner.EthAddress, testTokens);
+            return !string.IsNullOrEmpty(StartInteraction().GetTokenName(Deployment.TokenAddress));
         }
 
-        public void MintTestTokens(EthAddress ethAddress, TestToken testTokens)
+        public string MintTestTokens(IHasEthAddress owner, TestToken testTokens)
         {
-            var interaction = new ContractInteractions(log, gethNode);
-            interaction.MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
+            return MintTestTokens(owner.EthAddress, testTokens);
+        }
+
+        public string MintTestTokens(EthAddress ethAddress, TestToken testTokens)
+        {
+            return StartInteraction().MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
         }
 
         public TestToken GetTestTokenBalance(IHasEthAddress owner)
@@ -45,9 +73,108 @@ namespace CodexContractsPlugin
 
         public TestToken GetTestTokenBalance(EthAddress ethAddress)
         {
-            var interaction = new ContractInteractions(log, gethNode);
-            var balance = interaction.GetBalance(Deployment.TokenAddress, ethAddress.Address);
+            var balance = StartInteraction().GetBalance(Deployment.TokenAddress, ethAddress.Address);
             return balance.TestTokens();
         }
+
+        public Request[] GetStorageRequests(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<StorageRequestedEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            var i = StartInteraction();
+            return events
+                    .Select(e =>
+                    {
+                        var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId);
+                        var result = requestEvent.ReturnValue1;
+                        result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                        result.RequestId = e.Event.RequestId;
+                        return result;
+                    })
+                    .ToArray();
+        }
+
+        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<RequestCancelledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<SlotFilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<SlotFreedEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
+        {
+            var encoder = new ABIEncode();
+            var encoded = encoder.GetABIEncoded(
+                new ABIValue("bytes32", storageRequest.RequestId),
+                new ABIValue("uint256", slotIndex.ToBig())
+            );
+
+            var hashed = Sha3Keccack.Current.CalculateHash(encoded);
+
+            var func = new GetHostFunction
+            {
+                SlotId = hashed
+            };
+            var address = gethNode.Call<GetHostFunction, string>(Deployment.MarketplaceAddress, func);
+            if (string.IsNullOrEmpty(address)) return null;
+            return new EthAddress(address);
+        }
+
+        public RequestState GetRequestState(Request request)
+        {
+            var func = new RequestStateFunction
+            {
+                RequestId = request.RequestId
+            };
+            return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
+        }
+
+        private EthAddress GetEthAddressFromTransaction(string transactionHash)
+        {
+            var transaction = gethNode.GetTransaction(transactionHash);
+            return new EthAddress(transaction.From);
+        }
+
+        private ContractInteractions StartInteraction()
+        {
+            return new ContractInteractions(log, gethNode);
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -1,11 +1,13 @@
 using GethPlugin;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
+using Logging;
 
 namespace CodexContractsPlugin
 {
     public class CodexContractsContainerRecipe : ContainerRecipeFactory
     {
-        public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests";
+        public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:latest-dist-tests";
 
         public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
         public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
@@ -17,8 +19,9 @@ namespace CodexContractsPlugin
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 
-            var containerPort = config.GethNode.StartResult.Container.GetContainerPort(GethContainerRecipe.HttpPortTag);
-            var address = containerPort.InternalAddress;
+            var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
+
+            SetSchedulingAffinity(notIn: "false");
 
             AddEnvVar("DISTTEST_NETWORK_URL", address.ToString());
             AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork");

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -1,6 +1,7 @@
 using Core;
 using GethPlugin;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using Utils;
 
@@ -32,7 +33,7 @@ namespace CodexContractsPlugin
             try
             {
                 var result = DeployContract(container, workflow, gethNode);
-                workflow.Stop(containers);
+                workflow.Stop(containers, waitTillStopped: false);
                 Log("Container stopped.");
                 return result;
             }
@@ -84,7 +85,7 @@ namespace CodexContractsPlugin
 
         private void WaitUntil(Func<bool> predicate)
         {
-            Time.WaitUntil(predicate, TimeSpan.FromMinutes(3), TimeSpan.FromSeconds(2));
+            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2));
         }
 
         private StartupConfig CreateStartupConfig(IGethNode gethNode)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -1,7 +1,9 @@
-using GethPlugin;
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
 using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
+using Nethereum.Hex.HexConvertors.Extensions;
 using NethereumWorkflow;
 using System.Numerics;
 
@@ -26,10 +28,26 @@ namespace CodexContractsPlugin
             return gethNode.Call<GetTokenFunction, string>(marketplaceAddress, function);
         }
 
-        public void MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
+        public string GetTokenName(string tokenAddress)
+        {
+            try
+            {
+                log.Debug(tokenAddress);
+                var function = new GetTokenNameFunction();
+
+                return gethNode.Call<GetTokenNameFunction, string>(tokenAddress, function);
+            }
+            catch (Exception ex)
+            {
+                log.Log("Failed to get token name: " + ex);
+                return string.Empty;
+            }
+        }
+
+        public string MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
         {
             log.Debug($"{amount} -> {address} (token: {tokenAddress})");
-            MintTokens(address.Address, amount, tokenAddress);
+            return MintTokens(address.Address, amount, tokenAddress);
         }
 
         public decimal GetBalance(string tokenAddress, string account)
@@ -43,6 +61,17 @@ namespace CodexContractsPlugin
             return gethNode.Call<GetTokenBalanceFunction, BigInteger>(tokenAddress, function).ToDecimal();
         }
 
+        public GetRequestOutputDTO GetRequest(string marketplaceAddress, byte[] requestId)
+        {
+
+            log.Debug($"({marketplaceAddress}) {requestId.ToHex(true)}");
+            var func = new GetRequestFunction
+            {
+                RequestId = requestId
+            };
+            return gethNode.Call<GetRequestFunction, GetRequestOutputDTO>(marketplaceAddress, func);
+        }
+
         public bool IsSynced(string marketplaceAddress, string marketplaceAbi)
         {
             log.Debug();
@@ -56,7 +85,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        private void MintTokens(string account, decimal amount, string tokenAddress)
+        private string MintTokens(string account, decimal amount, string tokenAddress)
         {
             log.Debug($"({tokenAddress}) {amount} --> {account}");
             if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
@@ -67,7 +96,7 @@ namespace CodexContractsPlugin
                 Amount = amount.ToBig()
             };
 
-            gethNode.SendTransaction(tokenAddress, function);
+            return gethNode.SendTransaction(tokenAddress, function);
         }
 
         private bool IsBlockNumberOK()
@@ -87,6 +116,11 @@ namespace CodexContractsPlugin
     {
     }
 
+    [Function("name", "string")]
+    public class GetTokenNameFunction : FunctionMessage
+    {
+    }
+
     [Function("mint")]
     public class MintTokensFunction : FunctionMessage
     {

ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -0,0 +1,35 @@
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
+using GethPlugin;
+
+namespace CodexContractsPlugin.Marketplace
+{
+    public partial class Request : RequestBase
+    {
+        public ulong BlockNumber { get; set; }
+        public byte[] RequestId { get; set; }
+
+        public EthAddress ClientAddress { get { return new EthAddress(Client); } }
+    }
+
+    public partial class RequestFulfilledEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+    }
+
+    public partial class RequestCancelledEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+    }
+
+    public partial class SlotFilledEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+        public EthAddress Host { get; set; }
+    }
+
+    public partial class SlotFreedEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+    }
+}
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexContractsPlugin/Marketplace/Marketplace.cs

@@ -0,0 +1,517 @@
+using Nethereum.ABI.FunctionEncoding.Attributes;
+using Nethereum.Contracts;
+using System.Numerics;
+
+// Generated code, do not modify.
+
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
+namespace CodexContractsPlugin.Marketplace
+{
+    public partial class ConfigFunction : ConfigFunctionBase { }
+
+    [Function("config", typeof(ConfigOutputDTO))]
+    public class ConfigFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class FillSlotFunction : FillSlotFunctionBase { }
+
+    [Function("fillSlot")]
+    public class FillSlotFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("uint256", "slotIndex", 2)]
+        public virtual BigInteger SlotIndex { get; set; }
+        [Parameter("bytes", "proof", 3)]
+        public virtual byte[] Proof { get; set; }
+    }
+
+    public partial class FreeSlotFunction : FreeSlotFunctionBase { }
+
+    [Function("freeSlot")]
+    public class FreeSlotFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class GetActiveSlotFunction : GetActiveSlotFunctionBase { }
+
+    [Function("getActiveSlot", typeof(GetActiveSlotOutputDTO))]
+    public class GetActiveSlotFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class GetChallengeFunction : GetChallengeFunctionBase { }
+
+    [Function("getChallenge", "bytes32")]
+    public class GetChallengeFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class GetHostFunction : GetHostFunctionBase { }
+
+    [Function("getHost", "address")]
+    public class GetHostFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class GetPointerFunction : GetPointerFunctionBase { }
+
+    [Function("getPointer", "uint8")]
+    public class GetPointerFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class GetRequestFunction : GetRequestFunctionBase { }
+
+    [Function("getRequest", typeof(GetRequestOutputDTO))]
+    public class GetRequestFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class IsProofRequiredFunction : IsProofRequiredFunctionBase { }
+
+    [Function("isProofRequired", "bool")]
+    public class IsProofRequiredFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class MarkProofAsMissingFunction : MarkProofAsMissingFunctionBase { }
+
+    [Function("markProofAsMissing")]
+    public class MarkProofAsMissingFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+        [Parameter("uint256", "period", 2)]
+        public virtual BigInteger Period { get; set; }
+    }
+
+    public partial class MissingProofsFunction : MissingProofsFunctionBase { }
+
+    [Function("missingProofs", "uint256")]
+    public class MissingProofsFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class MyRequestsFunction : MyRequestsFunctionBase { }
+
+    [Function("myRequests", "bytes32[]")]
+    public class MyRequestsFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class MySlotsFunction : MySlotsFunctionBase { }
+
+    [Function("mySlots", "bytes32[]")]
+    public class MySlotsFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class RequestEndFunction : RequestEndFunctionBase { }
+
+    [Function("requestEnd", "uint256")]
+    public class RequestEndFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestStateFunction : RequestStateFunctionBase { }
+
+    [Function("requestState", "uint8")]
+    public class RequestStateFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestStorageFunction : RequestStorageFunctionBase { }
+
+    [Function("requestStorage")]
+    public class RequestStorageFunctionBase : FunctionMessage
+    {
+        [Parameter("tuple", "request", 1)]
+        public virtual Request Request { get; set; }
+    }
+
+    public partial class SlotStateFunction : SlotStateFunctionBase { }
+
+    [Function("slotState", "uint8")]
+    public class SlotStateFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class SubmitProofFunction : SubmitProofFunctionBase { }
+
+    [Function("submitProof")]
+    public class SubmitProofFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+        [Parameter("bytes", "proof", 2)]
+        public virtual byte[] Proof { get; set; }
+    }
+
+    public partial class TokenFunction : TokenFunctionBase { }
+
+    [Function("token", "address")]
+    public class TokenFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class WillProofBeRequiredFunction : WillProofBeRequiredFunctionBase { }
+
+    [Function("willProofBeRequired", "bool")]
+    public class WillProofBeRequiredFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class WithdrawFundsFunction : WithdrawFundsFunctionBase { }
+
+    [Function("withdrawFunds")]
+    public class WithdrawFundsFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class ProofSubmittedEventDTO : ProofSubmittedEventDTOBase { }
+
+    [Event("ProofSubmitted")]
+    public class ProofSubmittedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "id", 1, false)]
+        public virtual byte[] Id { get; set; }
+        [Parameter("bytes", "proof", 2, false)]
+        public virtual byte[] Proof { get; set; }
+    }
+
+    public partial class RequestCancelledEventDTO : RequestCancelledEventDTOBase { }
+
+    [Event("RequestCancelled")]
+    public class RequestCancelledEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestFailedEventDTO : RequestFailedEventDTOBase { }
+
+    [Event("RequestFailed")]
+    public class RequestFailedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestFulfilledEventDTO : RequestFulfilledEventDTOBase { }
+
+    [Event("RequestFulfilled")]
+    public class RequestFulfilledEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class SlotFilledEventDTO : SlotFilledEventDTOBase { }
+
+    [Event("SlotFilled")]
+    public class SlotFilledEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("uint256", "slotIndex", 2, false)]
+        public virtual BigInteger SlotIndex { get; set; }
+    }
+
+    public partial class SlotFreedEventDTO : SlotFreedEventDTOBase { }
+
+    [Event("SlotFreed")]
+    public class SlotFreedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("uint256", "slotIndex", 2, false)]
+        public virtual BigInteger SlotIndex { get; set; }
+    }
+
+    public partial class StorageRequestedEventDTO : StorageRequestedEventDTOBase { }
+
+    [Event("StorageRequested")]
+    public class StorageRequestedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, false)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("tuple", "ask", 2, false)]
+        public virtual Ask Ask { get; set; }
+        [Parameter("uint256", "expiry", 3, false)]
+        public virtual BigInteger Expiry { get; set; }
+    }
+
+    public partial class ConfigOutputDTO : ConfigOutputDTOBase { }
+
+    [FunctionOutput]
+    public class ConfigOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("tuple", "collateral", 1)]
+        public virtual CollateralConfig Collateral { get; set; }
+        [Parameter("tuple", "proofs", 2)]
+        public virtual ProofConfig Proofs { get; set; }
+    }
+
+
+
+
+
+    public partial class GetActiveSlotOutputDTO : GetActiveSlotOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetActiveSlotOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("tuple", "", 1)]
+        public virtual ActiveSlot ReturnValue1 { get; set; }
+    }
+
+    public partial class GetChallengeOutputDTO : GetChallengeOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetChallengeOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bytes32", "", 1)]
+        public virtual byte[] ReturnValue1 { get; set; }
+    }
+
+    public partial class GetHostOutputDTO : GetHostOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetHostOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("address", "", 1)]
+        public virtual string ReturnValue1 { get; set; }
+    }
+
+    public partial class GetPointerOutputDTO : GetPointerOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetPointerOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint8", "", 1)]
+        public virtual byte ReturnValue1 { get; set; }
+    }
+
+    public partial class GetRequestOutputDTO : GetRequestOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetRequestOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("tuple", "", 1)]
+        public virtual Request ReturnValue1 { get; set; }
+    }
+
+    public partial class IsProofRequiredOutputDTO : IsProofRequiredOutputDTOBase { }
+
+    [FunctionOutput]
+    public class IsProofRequiredOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bool", "", 1)]
+        public virtual bool ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class MissingProofsOutputDTO : MissingProofsOutputDTOBase { }
+
+    [FunctionOutput]
+    public class MissingProofsOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint256", "", 1)]
+        public virtual BigInteger ReturnValue1 { get; set; }
+    }
+
+    public partial class MyRequestsOutputDTO : MyRequestsOutputDTOBase { }
+
+    [FunctionOutput]
+    public class MyRequestsOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bytes32[]", "", 1)]
+        public virtual List<byte[]> ReturnValue1 { get; set; }
+    }
+
+    public partial class MySlotsOutputDTO : MySlotsOutputDTOBase { }
+
+    [FunctionOutput]
+    public class MySlotsOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bytes32[]", "", 1)]
+        public virtual List<byte[]> ReturnValue1 { get; set; }
+    }
+
+    public partial class RequestEndOutputDTO : RequestEndOutputDTOBase { }
+
+    [FunctionOutput]
+    public class RequestEndOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint256", "", 1)]
+        public virtual BigInteger ReturnValue1 { get; set; }
+    }
+
+    public partial class RequestStateOutputDTO : RequestStateOutputDTOBase { }
+
+    [FunctionOutput]
+    public class RequestStateOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint8", "", 1)]
+        public virtual byte ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class SlotStateOutputDTO : SlotStateOutputDTOBase { }
+
+    [FunctionOutput]
+    public class SlotStateOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint8", "", 1)]
+        public virtual byte ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class TokenOutputDTO : TokenOutputDTOBase { }
+
+    [FunctionOutput]
+    public class TokenOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("address", "", 1)]
+        public virtual string ReturnValue1 { get; set; }
+    }
+
+    public partial class WillProofBeRequiredOutputDTO : WillProofBeRequiredOutputDTOBase { }
+
+    [FunctionOutput]
+    public class WillProofBeRequiredOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bool", "", 1)]
+        public virtual bool ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class CollateralConfig : CollateralConfigBase { }
+
+    public class CollateralConfigBase
+    {
+        [Parameter("uint8", "repairRewardPercentage", 1)]
+        public virtual byte RepairRewardPercentage { get; set; }
+        [Parameter("uint8", "maxNumberOfSlashes", 2)]
+        public virtual byte MaxNumberOfSlashes { get; set; }
+        [Parameter("uint16", "slashCriterion", 3)]
+        public virtual ushort SlashCriterion { get; set; }
+        [Parameter("uint8", "slashPercentage", 4)]
+        public virtual byte SlashPercentage { get; set; }
+    }
+
+    public partial class ProofConfig : ProofConfigBase { }
+
+    public class ProofConfigBase
+    {
+        [Parameter("uint256", "period", 1)]
+        public virtual BigInteger Period { get; set; }
+        [Parameter("uint256", "timeout", 2)]
+        public virtual BigInteger Timeout { get; set; }
+        [Parameter("uint8", "downtime", 3)]
+        public virtual byte Downtime { get; set; }
+    }
+
+    public partial class MarketplaceConfig : MarketplaceConfigBase { }
+
+    public class MarketplaceConfigBase
+    {
+        [Parameter("tuple", "collateral", 1)]
+        public virtual CollateralConfig Collateral { get; set; }
+        [Parameter("tuple", "proofs", 2)]
+        public virtual ProofConfig Proofs { get; set; }
+    }
+
+    public partial class Ask : AskBase { }
+
+    public class AskBase
+    {
+        [Parameter("uint64", "slots", 1)]
+        public virtual ulong Slots { get; set; }
+        [Parameter("uint256", "slotSize", 2)]
+        public virtual BigInteger SlotSize { get; set; }
+        [Parameter("uint256", "duration", 3)]
+        public virtual BigInteger Duration { get; set; }
+        [Parameter("uint256", "proofProbability", 4)]
+        public virtual BigInteger ProofProbability { get; set; }
+        [Parameter("uint256", "reward", 5)]
+        public virtual BigInteger Reward { get; set; }
+        [Parameter("uint256", "collateral", 6)]
+        public virtual BigInteger Collateral { get; set; }
+        [Parameter("uint64", "maxSlotLoss", 7)]
+        public virtual ulong MaxSlotLoss { get; set; }
+    }
+
+    public partial class Content : ContentBase { }
+
+    public class ContentBase
+    {
+        [Parameter("string", "cid", 1)]
+        public virtual string Cid { get; set; }
+        [Parameter("bytes32", "merkleRoot", 2)]
+        public virtual byte[] MerkleRoot { get; set; }
+    }
+
+    public partial class Request : RequestBase { }
+
+    public class RequestBase
+    {
+        [Parameter("address", "client", 1)]
+        public virtual string Client { get; set; }
+        [Parameter("tuple", "ask", 2)]
+        public virtual Ask Ask { get; set; }
+        [Parameter("tuple", "content", 3)]
+        public virtual Content Content { get; set; }
+        [Parameter("uint256", "expiry", 4)]
+        public virtual BigInteger Expiry { get; set; }
+        [Parameter("bytes32", "nonce", 5)]
+        public virtual byte[] Nonce { get; set; }
+    }
+
+    public partial class ActiveSlot : ActiveSlotBase { }
+
+    public class ActiveSlotBase
+    {
+        [Parameter("tuple", "request", 1)]
+        public virtual Request Request { get; set; }
+        [Parameter("uint256", "slotIndex", 2)]
+        public virtual BigInteger SlotIndex { get; set; }
+    }
+}
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexContractsPlugin/Marketplace/README.md

@@ -0,0 +1 @@
+This code was generated using the Nethereum code generator, here: http://playground.nethereum.com

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs

@@ -1,5 +1,6 @@
 using Core;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace CodexDiscordBotPlugin
 {
@@ -28,19 +29,18 @@ namespace CodexDiscordBotPlugin
         {
         }
 
-        public RunningContainer Deploy(DiscordBotStartupConfig config)
+        public RunningContainers Deploy(DiscordBotStartupConfig config)
         {
             var workflow = tools.CreateWorkflow();
             return StartContainer(workflow, config);
         }
 
-        private RunningContainer StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
+        private RunningContainers StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
         {
             var startupConfig = new StartupConfig();
             startupConfig.NameOverride = config.Name;
             startupConfig.Add(config);
-            var rc = workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig);
-            return rc.Containers.Single();
+            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig);
         }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs

@@ -1,11 +1,11 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace CodexDiscordBotPlugin
 {
     public static class CoreInterfaceExtensions
     {
-        public static RunningContainer DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
+        public static RunningContainers DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
         {
             return Plugin(ci).Deploy(config);
         }

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
 using Utils;
 
 namespace CodexDiscordBotPlugin
@@ -12,16 +13,30 @@ namespace CodexDiscordBotPlugin
         {
             var config = startupConfig.Get<DiscordBotStartupConfig>();
 
+            SetSchedulingAffinity(notIn: "false");
+
             AddEnvVar("TOKEN", config.Token);
             AddEnvVar("SERVERNAME", config.ServerName);
             AddEnvVar("ADMINROLE", config.AdminRoleName);
             AddEnvVar("ADMINCHANNELNAME", config.AdminChannelName);
+            AddEnvVar("KUBECONFIG", "/opt/kubeconfig.yaml");
+            AddEnvVar("KUBENAMESPACE", config.KubeNamespace);
+
+            var gethInfo = config.GethInfo;
+            AddEnvVar("GETH_HOST", gethInfo.Host);
+            AddEnvVar("GETH_HTTP_PORT", gethInfo.Port.ToString());
+            AddEnvVar("GETH_PRIVATE_KEY", gethInfo.PrivKey);
+            AddEnvVar("CODEXCONTRACTS_MARKETPLACEADDRESS", gethInfo.MarketplaceAddress);
+            AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress);
+            AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi);
 
             if (!string.IsNullOrEmpty(config.DataPath))
             {
                 AddEnvVar("DATAPATH", config.DataPath);
                 AddVolume(config.DataPath, 1.GB());
             }
+
+            AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml", secret: "discordbot-sa-kubeconfig");
         }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs

@@ -2,13 +2,15 @@
 {
     public class DiscordBotStartupConfig
     {
-        public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName)
+        public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName, string kubeNamespace, DiscordBotGethInfo gethInfo)
         {
             Name = name;
             Token = token;
             ServerName = serverName;
             AdminRoleName = adminRoleName;
             AdminChannelName = adminChannelName;
+            KubeNamespace = kubeNamespace;
+            GethInfo = gethInfo;
         }
 
         public string Name { get; }
@@ -16,6 +18,28 @@
         public string ServerName { get; }
         public string AdminRoleName { get; }
         public string AdminChannelName { get; }
+        public string KubeNamespace { get; }
+        public DiscordBotGethInfo GethInfo { get; }
         public string? DataPath { get; set; }
     }
+
+    public class DiscordBotGethInfo
+    {
+        public DiscordBotGethInfo(string host, int port, string privKey, string marketplaceAddress, string tokenAddress, string abi)
+        {
+            Host = host;
+            Port = port;
+            PrivKey = privKey;
+            MarketplaceAddress = marketplaceAddress;
+            TokenAddress = tokenAddress;
+            Abi = abi;
+        }
+
+        public string Host { get; }
+        public int Port { get; }
+        public string PrivKey { get; }
+        public string MarketplaceAddress { get; }
+        public string TokenAddress { get; }
+        public string Abi { get; }
+    }
 }

ProjectPlugins/CodexPlugin/ApiChecker.cs

@@ -0,0 +1,92 @@
+using Core;
+using KubernetesWorkflow.Types;
+using Logging;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace CodexPlugin
+{
+    public class ApiChecker
+    {
+        // <INSERT-OPENAPI-YAML-HASH>
+        private const string OpenApiYamlHash = "DC-90-1B-63-76-1B-92-01-05-65-33-DA-17-C2-34-83-E1-2E-6C-A9-04-4D-68-ED-96-43-F5-E5-6A-00-0F-5F";
+        private const string OpenApiFilePath = "/codex/openapi.yaml";
+        private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";
+
+        private const bool Disable = false;
+
+        private const string Warning =
+            "Warning: CodexPlugin was unable to find the openapi.yaml file in the Codex container. Are you running an old version of Codex? " +
+            "Plugin will continue as normal, but API compatibility is not guaranteed!";
+
+        private const string Failure =
+            "Codex API compatibility check failed! " +
+            "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. Please update the openapi.yaml in " +
+            "'ProjectPlugins/CodexPlugin' and rebuild this project. If you wish to disable API compatibility checking, please set " +
+            $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/CodexPlugin/ApiChecker.cs'.";
+
+        private static bool checkPassed = false;
+
+        private readonly IPluginTools pluginTools;
+        private readonly ILog log;
+
+        public ApiChecker(IPluginTools pluginTools)
+        {
+            this.pluginTools = pluginTools;
+            log = pluginTools.GetLog();
+
+            if (string.IsNullOrEmpty(OpenApiYamlHash)) throw new Exception("OpenAPI yaml hash was not inserted by pre-build trigger.");
+        }
+
+        public void CheckCompatibility(RunningContainers[] containers)
+        {
+            if (checkPassed) return;
+
+            Log("CodexPlugin is checking API compatibility...");
+
+            if (Disable || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(DisableEnvironmentVariable)))
+            {
+                Log("API compatibility checking has been disabled.");
+                checkPassed = true;
+                return;
+            }
+
+            var workflow = pluginTools.CreateWorkflow();
+            var container = containers.First().Containers.First();
+            var containerApi = workflow.ExecuteCommand(container, "cat", OpenApiFilePath);
+
+            if (string.IsNullOrEmpty(containerApi))
+            {
+                log.Error(Warning);
+
+                checkPassed = true;
+                return;
+            }
+
+            var containerHash = Hash(containerApi);
+            if (containerHash == OpenApiYamlHash)
+            {
+                Log("API compatibility check passed.");
+                checkPassed = true;
+                return;
+            }
+
+            log.Error(Failure);
+            throw new Exception(Failure);
+        }
+
+        private string Hash(string file)
+        {
+            var fileBytes = Encoding.ASCII.GetBytes(file
+                .Replace(Environment.NewLine, ""));
+            var sha = SHA256.Create();
+            var hash = sha.ComputeHash(fileBytes);
+            return BitConverter.ToString(hash);
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+    }
+}

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -1,5 +1,7 @@
-using Core;
+using CodexOpenApi;
+using Core;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Utils;
 
 namespace CodexPlugin
@@ -7,6 +9,7 @@ namespace CodexPlugin
     public class CodexAccess : ILogHandler
     {
         private readonly IPluginTools tools;
+        private readonly Mapper mapper = new Mapper();
         private bool hasContainerCrashed;
 
         public CodexAccess(IPluginTools tools, RunningContainer container, CrashWatcher crashWatcher)
@@ -22,74 +25,72 @@ namespace CodexPlugin
         public RunningContainer Container { get; }
         public CrashWatcher CrashWatcher { get; }
 
-        public CodexDebugResponse GetDebugInfo()
+        public DebugInfo GetDebugInfo()
         {
-            return Http().HttpGetJson<CodexDebugResponse>("debug/info");
+            return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
         }
 
-        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        public DebugPeer GetDebugPeer(string peerId)
         {
-            var http = Http();
-            var str = http.HttpGetString($"debug/peer/{peerId}");
+            // Cannot use openAPI: debug/peer endpoint is not specified there.
+            var endpoint = GetEndpoint();
+            var str = endpoint.HttpGetString($"debug/peer/{peerId}");
 
             if (str.ToLowerInvariant() == "unable to find peer!")
             {
-                return new CodexDebugPeerResponse
+                return new DebugPeer
                 {
                     IsPeerFound = false
                 };
             }
 
-            var result = http.Deserialize<CodexDebugPeerResponse>(str);
+            var result = endpoint.Deserialize<DebugPeer>(str);
             result.IsPeerFound = true;
             return result;
         }
 
-        public CodexDebugBlockExchangeResponse GetDebugBlockExchange()
+        public void ConnectToPeer(string peerId, string[] peerMultiAddresses)
         {
-            return Http().HttpGetJson<CodexDebugBlockExchangeResponse>("debug/blockexchange");
-        }
-
-        public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
-        {
-            return LongHttp().HttpGetJson<CodexDebugRepoStoreResponse[]>("debug/repostore");
-        }
-
-        public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
-        {
-            return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");
+            OnCodex(api =>
+            {
+                Time.Wait(api.ConnectPeerAsync(peerId, peerMultiAddresses));
+                return Task.FromResult(string.Empty);
+            });
         }
 
         public string UploadFile(FileStream fileStream)
         {
-            // private const string UploadFailedMessage = "Unable to store block";
-
-            return Http().HttpPostStream("upload", fileStream);
+            return OnCodex(api => api.UploadAsync(fileStream));
         }
 
         public Stream DownloadFile(string contentId)
         {
-            return Http().HttpGetStream("download/" + contentId);
+            var fileResponse = OnCodex(api => api.DownloadNetworkAsync(contentId));
+            if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
+            return fileResponse.Stream;
         }
 
-        public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
+        public LocalDatasetList LocalFiles()
         {
-            return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
+            return mapper.Map(OnCodex(api => api.ListDataAsync()));
         }
 
-        public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
+        public StorageAvailability SalesAvailability(StorageAvailability request)
         {
-            return Http().HttpPostJson($"storage/request/{contentId}", request);
+            var body = mapper.Map(request);
+            var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
+            return mapper.Map(read);
         }
 
-        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
+        public string RequestStorage(StoragePurchaseRequest request)
         {
-            return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
+            var body = mapper.Map(request);
+            return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
         }
 
-        public string ConnectToPeer(string peerId, string peerMultiAddress)
+        public StoragePurchase GetPurchaseStatus(string purchaseId)
         {
-            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
+            return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
         }
 
         public string GetName()
@@ -97,19 +98,35 @@ namespace CodexPlugin
             return Container.Name;
         }
 
-        private IHttp Http()
+        public PodInfo GetPodInfo()
         {
-            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
+            var workflow = tools.CreateWorkflow();
+            return workflow.GetPodInfo(Container);
         }
 
-        private IHttp LongHttp()
+        private T OnCodex<T>(Func<CodexApi, Task<T>> action)
         {
-            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name);
+            var address = GetAddress();
+            var result = tools.CreateHttp(CheckContainerCrashed)
+                .OnClient(client =>
+            {
+                var api = new CodexApi(client);
+                api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
+                return Time.Wait(action(api));
+            });
+            return result;
+        }
+
+        private IEndpoint GetEndpoint()
+        {
+            return tools
+                .CreateHttp(CheckContainerCrashed)
+                .CreateEndpoint(GetAddress(), "/api/codex/v1/", Container.Name);
         }
 
         private Address GetAddress()
         {
-            return Container.GetAddress(CodexContainerRecipe.ApiPortTag);
+            return Container.GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
         }
 
         private void CheckContainerCrashed(HttpClient client)

ProjectPlugins/CodexPlugin/CodexApiTypes.cs

@@ -1,173 +0,0 @@
-using Newtonsoft.Json;
-
-namespace CodexPlugin
-{
-    public class CodexDebugResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string[] addrs { get; set; } = new string[0];
-        public string repo { get; set; } = string.Empty;
-        public string spr { get; set; } = string.Empty;
-        public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
-        public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
-        public CodexDebugVersionResponse codex { get; set; } = new();
-        public CodexDebugTableResponse table { get; set; } = new();
-    }
-
-    public class CodexDebugFutures
-    {
-        public int futures { get; set; }
-    }
-
-    public class CodexDebugTableResponse
-    {
-        public CodexDebugTableNodeResponse localNode { get; set; } = new();
-        public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
-    }
-
-    public class CodexDebugTableNodeResponse
-    {
-        public string nodeId { get; set; } = string.Empty;
-        public string peerId { get; set; } = string.Empty;
-        public string record { get; set; } = string.Empty;
-        public string address { get; set; } = string.Empty;
-        public bool seen { get; set; }
-    }
-
-    public class EnginePeerResponse
-    {
-        public string peerId { get; set; } = string.Empty;
-        public EnginePeerContextResponse context { get; set; } = new();
-    }
-
-    public class EnginePeerContextResponse
-    {
-        public int blocks { get; set; } = 0;
-        public int peerWants { get; set; } = 0;
-        public int exchanged { get; set; } = 0;
-        public string lastExchange { get; set; } = string.Empty;
-    }
-
-    public class SwitchPeerResponse
-    {
-        public string peerId { get; set; } = string.Empty;
-        public string key { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugVersionResponse
-    {
-        public string version { get; set; } = string.Empty;
-        public string revision { get; set; } = string.Empty;
-
-        public bool IsValid()
-        {
-            return !string.IsNullOrEmpty(version) && !string.IsNullOrEmpty(revision);
-        }
-
-        public override string ToString()
-        {
-            return JsonConvert.SerializeObject(this);
-        }
-    }
-
-    public class CodexDebugPeerResponse
-    {
-        public bool IsPeerFound { get; set; }
-
-        public string peerId { get; set; } = string.Empty;
-        public long seqNo { get; set; }
-        public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
-    }
-
-    public class CodexDebugPeerAddressResponse
-    {
-        public string address { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugThresholdBreaches
-    {
-        public string[] breaches { get; set; } = Array.Empty<string>();
-    }
-
-    public class CodexSalesAvailabilityRequest
-    {
-        public string size { get; set; } = string.Empty;
-        public string duration { get; set; } = string.Empty;
-        public string minPrice { get; set; } = string.Empty;
-        public string maxCollateral { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesAvailabilityResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string size { get; set; } = string.Empty;
-        public string duration { get; set; } = string.Empty;
-        public string minPrice { get; set; } = string.Empty;
-        public string maxCollateral { get; set; } = string.Empty;
-    }
-
-    public class CodexSalesRequestStorageRequest
-    {
-        public string duration { get; set; } = string.Empty;
-        public string proofProbability { get; set; } = string.Empty;
-        public string reward { get; set; } = string.Empty;
-        public string collateral { get; set; } = string.Empty;
-        public string? expiry { get; set; }
-        public uint? nodes { get; set; }
-        public uint? tolerance { get; set; }
-    }
-
-    public class CodexStoragePurchase
-    {
-        public string state { get; set; } = string.Empty;
-        public string error { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugBlockExchangeResponse
-    {
-        public CodexDebugBlockExchangeResponsePeer[] peers { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeer>();
-        public int taskQueue { get; set; }
-        public int pendingBlocks { get; set; }
-
-        public override string ToString()
-        {
-            if (peers.Length == 0 && taskQueue == 0 && pendingBlocks == 0) return "all-empty";
-
-            return $"taskqueue: {taskQueue} pendingblocks: {pendingBlocks} peers: {string.Join(",", peers.Select(p => p.ToString()))}";
-        }
-    }
-
-    public class CodexDebugBlockExchangeResponsePeer
-    {
-        public string peerid { get; set; } = string.Empty;
-        public CodexDebugBlockExchangeResponsePeerHasBlock[] hasBlocks { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeerHasBlock>();
-        public CodexDebugBlockExchangeResponsePeerWant[] wants { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeerWant>();
-        public int exchanged { get; set; }
-
-        public override string ToString()
-        {
-            return $"(blocks:{hasBlocks.Length} wants:{wants.Length})";
-        }
-    }
-
-    public class CodexDebugBlockExchangeResponsePeerHasBlock
-    {
-        public string cid { get; set; } = string.Empty;
-        public bool have { get; set; }
-        public string price { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugBlockExchangeResponsePeerWant
-    {
-        public string block { get; set; } = string.Empty;
-        public int priority { get; set; }
-        public bool cancel { get; set; }
-        public string wantType { get; set; } = string.Empty;
-        public bool sendDontHave { get; set; }
-    }
-
-    public class CodexDebugRepoStoreResponse
-    {
-        public string cid { get; set; } = string.Empty; 
-    }
-}

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -1,13 +1,13 @@
-using KubernetesWorkflow;
+using GethPlugin;
+using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
 using Utils;
 
 namespace CodexPlugin
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter();
-
-        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-cd280d4-dist-tests";
         public const string ApiPortTag = "codex_api_port";
         public const string ListenPortTag = "codex_listen_port";
         public const string MetricsPortTag = "codex_metrics_port";
@@ -25,7 +25,10 @@ namespace CodexPlugin
         protected override void Initialize(StartupConfig startupConfig)
         {
             SetResourcesRequest(milliCPUs: 100, memory: 100.MB());
-            SetResourceLimits(milliCPUs: 4000, memory: 12.GB());
+            //SetResourceLimits(milliCPUs: 4000, memory: 12.GB());
+
+            SetSchedulingAffinity(notIn: "false");
+            SetSystemCriticalPriority();
 
             var config = startupConfig.Get<CodexStartupConfig>();
 
@@ -43,8 +46,9 @@ namespace CodexPlugin
 
             if (config.PublicTestNet != null)
             {
-                AddEnvVar("CODEX_NAT", config.PublicTestNet.PublicNatIP);
+                // This makes the node announce itself to its public IP address.
                 AddEnvVar("NAT_IP_AUTO", "false");
+                AddEnvVar("NAT_PUBLIC_IP_AUTO", PublicIpService.Address);
             }
             else
             {
@@ -94,21 +98,21 @@ namespace CodexPlugin
             {
                 var mconfig = config.MarketplaceConfig;
                 var gethStart = mconfig.GethNode.StartResult;
-                var ip = gethStart.Container.Pod.PodInfo.Ip;
-                var port = gethStart.WsPort.Number;
+                var wsAddress = gethStart.Container.GetInternalAddress(GethContainerRecipe.WsPortTag);
                 var marketplaceAddress = mconfig.CodexContracts.Deployment.MarketplaceAddress;
 
-                AddEnvVar("CODEX_ETH_PROVIDER", $"ws://{ip}:{port}");
+                AddEnvVar("CODEX_ETH_PROVIDER", $"{wsAddress.Host.Replace("http://", "ws://")}:{wsAddress.Port}");
                 AddEnvVar("CODEX_MARKETPLACE_ADDRESS", marketplaceAddress);
-                AddEnvVar("CODEX_PERSISTENCE", "true");
+
+                var marketplaceSetup = config.MarketplaceConfig.MarketplaceSetup;
 
                 // Custom scripting in the Codex test image will write this variable to a private-key file,
                 // and pass the correct filename to Codex.
-                var mStart = marketplaceStarter.Start();
-                AddEnvVar("PRIV_KEY", mStart.PrivateKey);
-                Additional(mStart);
+                AddEnvVar("PRIV_KEY", marketplaceSetup.EthAccount.PrivateKey);
+                Additional(marketplaceSetup.EthAccount);
 
-                if (config.MarketplaceConfig.IsValidator)
+                SetCommandOverride(marketplaceSetup);
+                if (marketplaceSetup.IsValidator)
                 {
                    AddEnvVar("CODEX_VALIDATOR", "true");
                 }
@@ -120,6 +124,18 @@ namespace CodexPlugin
             }
         }
 
+        private void SetCommandOverride(MarketplaceSetup ms)
+        {
+            if (ms.IsStorageNode)
+            {
+                OverrideCommand("bash", "/docker-entrypoint.sh", "codex", "persistence", "prover");
+            }
+            else
+            {
+                OverrideCommand("bash", "/docker-entrypoint.sh", "codex", "persistence");
+            }
+        }
+
         private Port CreateApiPort(CodexStartupConfig config, string tag)
         {
             if (config.PublicTestNet == null) return AddExposedPort(tag);

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -1,13 +1,17 @@
 using CodexContractsPlugin;
 using GethPlugin;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {
     public class CodexDeployment
     {
-        public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment, CodexContractsDeployment codexContractsDeployment, RunningContainer? prometheusContainer, RunningContainer? discordBotContainer, DeploymentMetadata metadata)
+        public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
+            CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer,
+            RunningContainers? discordBotContainer, DeploymentMetadata metadata,
+            String id)
         {
+            Id = id;
             CodexInstances = codexInstances;
             GethDeployment = gethDeployment;
             CodexContractsDeployment = codexContractsDeployment;
@@ -16,29 +20,33 @@ namespace CodexPlugin
             Metadata = metadata;
         }
 
+        public String Id { get; }
         public CodexInstance[] CodexInstances { get; }
         public GethDeployment GethDeployment { get; }
         public CodexContractsDeployment CodexContractsDeployment { get; }
-        public RunningContainer? PrometheusContainer { get; }
-        public RunningContainer? DiscordBotContainer { get; }
+        public RunningContainers? PrometheusContainer { get; }
+        public RunningContainers? DiscordBotContainer { get; }
         public DeploymentMetadata Metadata { get; }
     }
 
     public class CodexInstance
     {
-        public CodexInstance(RunningContainer container, CodexDebugResponse info)
+        public CodexInstance(RunningContainers containers, DebugInfo info)
         {
-            Container = container;
+            Containers = containers;
             Info = info;
         }
 
-        public RunningContainer Container { get; }
-        public CodexDebugResponse Info { get; }
+        public RunningContainers Containers { get; }
+        public DebugInfo Info { get; }
     }
 
     public class DeploymentMetadata
     {
-        public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
+        public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace,
+            int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel,
+            int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI,
+            int blockMN)
         {
             Name = name;
             StartUtc = startUtc;
@@ -73,4 +81,4 @@ namespace CodexPlugin
         public int BlockMI { get; }
         public int BlockMN { get; }
     }
-}
+}

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -2,6 +2,7 @@
 using FileUtils;
 using GethPlugin;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using MetricsPlugin;
 using Utils;
@@ -11,26 +12,26 @@ namespace CodexPlugin
     public interface ICodexNode : IHasContainer, IHasMetricsScrapeTarget, IHasEthAddress
     {
         string GetName();
-        CodexDebugResponse GetDebugInfo();
-        CodexDebugPeerResponse GetDebugPeer(string peerId);
-        // These debug methods are not available in master-line Codex. Use only for custom builds.
-        //CodexDebugBlockExchangeResponse GetDebugBlockExchange();
-        //CodexDebugRepoStoreResponse[] GetDebugRepoStore();
+        DebugInfo GetDebugInfo();
+        DebugPeer GetDebugPeer(string peerId);
         ContentId UploadFile(TrackedFile file);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
+        LocalDatasetList LocalFiles();
         void ConnectToPeer(ICodexNode node);
-        CodexDebugVersionResponse Version { get; }
+        DebugInfoVersion Version { get; }
         IMarketplaceAccess Marketplace { get; }
         CrashWatcher CrashWatcher { get; }
-        void Stop();
+        PodInfo GetPodInfo();
+        ITransferSpeeds TransferSpeeds { get; }
+        void Stop(bool waitTillStopped);
     }
 
     public class CodexNode : ICodexNode
     {
-        private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
         private const string UploadFailedMessage = "Unable to store block";
         private readonly IPluginTools tools;
         private readonly EthAddress? ethAddress;
+        private readonly TransferSpeeds transferSpeeds;
 
         public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAddress? ethAddress)
         {
@@ -39,7 +40,8 @@ namespace CodexPlugin
             CodexAccess = codexAccess;
             Group = group;
             Marketplace = marketplaceAccess;
-            Version = new CodexDebugVersionResponse();
+            Version = new DebugInfoVersion();
+            transferSpeeds = new TransferSpeeds();
         }
 
         public RunningContainer Container { get { return CodexAccess.Container; } }
@@ -47,16 +49,17 @@ namespace CodexPlugin
         public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; }
         public CodexNodeGroup Group { get; }
         public IMarketplaceAccess Marketplace { get; }
-        public CodexDebugVersionResponse Version { get; private set; }
+        public DebugInfoVersion Version { get; private set; }
+        public ITransferSpeeds TransferSpeeds { get => transferSpeeds; }
+
         public IMetricsScrapeTarget MetricsScrapeTarget
         {
             get
             {
-                var port = CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag);
-                if (port == null) throw new Exception("Metrics is not available for this Codex node. Please start it with the option '.EnableMetrics()' to enable it.");
-                return new MetricsScrapeTarget(CodexAccess.Container, port);
+                return new MetricsScrapeTarget(CodexAccess.Container, CodexContainerRecipe.MetricsPortTag);
             }
         }
+
         public EthAddress EthAddress 
         {
             get
@@ -71,40 +74,33 @@ namespace CodexPlugin
             return CodexAccess.Container.Name;
         }
 
-        public CodexDebugResponse GetDebugInfo()
+        public DebugInfo GetDebugInfo()
         {
             var debugInfo = CodexAccess.GetDebugInfo();
-            var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
-            Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
+            var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId));
+            Log($"Got DebugInfo with id: '{debugInfo.Id}'. This node knows: {known}");
             return debugInfo;
         }
 
-        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        public DebugPeer GetDebugPeer(string peerId)
         {
             return CodexAccess.GetDebugPeer(peerId);
         }
 
-        public CodexDebugBlockExchangeResponse GetDebugBlockExchange()
-        {
-            return CodexAccess.GetDebugBlockExchange();
-        }
-
-        public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
-        {
-            return CodexAccess.GetDebugRepoStore();
-        }
-
         public ContentId UploadFile(TrackedFile file)
         {
             using var fileStream = File.OpenRead(file.Filename);
 
             var logMessage = $"Uploading file {file.Describe()}...";
             Log(logMessage);
-            var response = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
+            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
             {
                 return CodexAccess.UploadFile(fileStream);
             });
 
+            var response = measurement.Value;
+            transferSpeeds.AddUploadSample(file.GetFilesize(), measurement.Duration);
+
             if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
             if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
 
@@ -117,56 +113,69 @@ namespace CodexPlugin
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
             Log(logMessage);
             var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
-            Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file));
+            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file));
+            transferSpeeds.AddDownloadSample(file.GetFilesize(), measurement);
             Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
             return file;
         }
 
+        public LocalDatasetList LocalFiles()
+        {
+            return CodexAccess.LocalFiles();
+        }
+
         public void ConnectToPeer(ICodexNode node)
         {
             var peer = (CodexNode)node;
 
             Log($"Connecting to peer {peer.GetName()}...");
             var peerInfo = node.GetDebugInfo();
-            var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
+            CodexAccess.ConnectToPeer(peerInfo.Id, GetPeerMultiAddresses(peer, peerInfo));
 
-            FrameworkAssert.That(response == SuccessfullyConnectedMessage, "Unable to connect codex nodes.");
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public void Stop()
+        public PodInfo GetPodInfo()
+        {
+            return CodexAccess.GetPodInfo();
+        }
+
+        public void Stop(bool waitTillStopped)
         {
             if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
                 "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
                 "available for codex-nodes in groups of 1.");
 
-            Group.BringOffline();
+            Group.BringOffline(waitTillStopped);
         }
 
         public void EnsureOnlineGetVersionResponse()
         {
             var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
-            var nodePeerId = debugInfo.id;
+            var nodePeerId = debugInfo.Id;
             var nodeName = CodexAccess.Container.Name;
 
-            if (!debugInfo.codex.IsValid())
+            if (!debugInfo.Version.IsValid())
             {
-                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}");
+                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}");
             }
 
-            //lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
-            //lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
-            Version = debugInfo.codex;
+            var log = tools.GetLog();
+            log.AddStringReplace(nodePeerId, nodeName);
+            log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName);
+            Version = debugInfo.Version;
         }
 
-        private string GetPeerMultiAddress(CodexNode peer, CodexDebugResponse peerInfo)
+        private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo)
         {
-            var multiAddress = peerInfo.addrs.First();
-            // Todo: Is there a case where First address in list is not the way?
-
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip);
+            var workflow = tools.CreateWorkflow();
+            var podInfo = workflow.GetPodInfo(peer.Container);
+
+            return peerInfo.Addrs.Select(a => a
+                .Replace("0.0.0.0", podInfo.Ip))
+                .ToArray();
         }
 
         private void DownloadToFile(string contentId, TrackedFile file)
@@ -189,14 +198,4 @@ namespace CodexPlugin
             tools.GetLog().Log($"{GetName()}: {msg}");
         }
     }
-
-    public class ContentId
-    {
-        public ContentId(string id)
-        {
-            Id = id;
-        }
-
-        public string Id { get; }
-    }
 }

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -1,6 +1,7 @@
 using Core;
 using GethPlugin;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {
@@ -34,9 +35,9 @@ namespace CodexPlugin
 
         private EthAddress? GetEthAddress(CodexAccess access)
         {
-            var mStart = access.Container.Recipe.Additionals.Get<MarketplaceStartResults>();
-            if (mStart == null) return null;
-            return mStart.EthAddress;
+            var ethAccount = access.Container.Recipe.Additionals.Get<EthAccount>();
+            if (ethAccount == null) return null;
+            return ethAccount.EthAddress;
         }
 
         public CrashWatcher CreateCrashWatcher(RunningContainer c)

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using MetricsPlugin;
 using System.Collections;
 
@@ -7,7 +7,7 @@ namespace CodexPlugin
 {
     public interface ICodexNodeGroup : IEnumerable<ICodexNode>, IHasManyMetricScrapeTargets
     {
-        void BringOffline();
+        void BringOffline(bool waitTillStopped);
         ICodexNode this[int index] { get; }
     }
 
@@ -20,7 +20,7 @@ namespace CodexPlugin
             this.starter = starter;
             Containers = containers;
             Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
-            Version = new CodexDebugVersionResponse();
+            Version = new DebugInfoVersion();
         }
 
         public ICodexNode this[int index]
@@ -31,9 +31,9 @@ namespace CodexPlugin
             }
         }
 
-        public void BringOffline()
+        public void BringOffline(bool waitTillStopped)
         {
-            starter.BringOffline(this);
+            starter.BringOffline(this, waitTillStopped);
             // Clear everything. Prevent accidental use.
             Nodes = Array.Empty<CodexNode>();
             Containers = null!;
@@ -41,7 +41,7 @@ namespace CodexPlugin
 
         public RunningContainers[] Containers { get; private set; }
         public CodexNode[] Nodes { get; private set; }
-        public CodexDebugVersionResponse Version { get; private set; }
+        public DebugInfoVersion Version { get; private set; }
         public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray();
 
         public IEnumerator<ICodexNode> GetEnumerator()
@@ -65,7 +65,7 @@ namespace CodexPlugin
             var versionResponses = Nodes.Select(n => n.Version);
 
             var first = versionResponses.First();
-            if (!versionResponses.All(v => v.version == first.version && v.revision == first.revision))
+            if (!versionResponses.All(v => v.Version == first.Version && v.Revision == first.Revision))
             {
                 throw new Exception("Inconsistent version information received from one or more Codex nodes: " +
                     string.Join(",", versionResponses.Select(v => v.ToString())));

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -1,5 +1,5 @@
-using Core;
-using KubernetesWorkflow;
+using Core;
+using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {
@@ -19,12 +19,13 @@ namespace CodexPlugin
 
         public void Announce()
         {
-            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}'");
+            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
         }
 
         public void AddMetadata(IAddMetadata metadata)
         {
             metadata.Add("codexid", codexStarter.GetCodexId());
+            metadata.Add("codexrevision", codexStarter.GetCodexRevision());
         }
 
         public void Decommission()
@@ -51,8 +52,8 @@ namespace CodexPlugin
             var mconfig = codexSetup.MarketplaceConfig;
             foreach (var node in result)
             {
-                mconfig.GethNode.SendEth(node, mconfig.InitialEth);
-                mconfig.CodexContracts.MintTestTokens(node, mconfig.InitialTokens);
+                mconfig.GethNode.SendEth(node, mconfig.MarketplaceSetup.InitialEth);
+                mconfig.CodexContracts.MintTestTokens(node, mconfig.MarketplaceSetup.InitialTestTokens);
             }
         }
 

ProjectPlugins/CodexPlugin/CodexPlugin.csproj

@@ -7,7 +7,23 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <None Remove="openapi.yaml" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <OpenApiReference Include="openapi.yaml" CodeGenerator="NSwagCSharp" Namespace="CodexOpenApi" ClassName="CodexApi" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.ApiDescription.Client" Version="7.0.2">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="NSwag.ApiDescription.Client" Version="13.18.2">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
   </ItemGroup>
 
   <ItemGroup>
@@ -18,4 +34,8 @@
     <ProjectReference Include="..\MetricsPlugin\MetricsPlugin.csproj" />
   </ItemGroup>
 
+  <Target Name="PreBuild" BeforeTargets="PreBuildEvent">
+    <Exec Command="dotnet run --project $(ProjectDir)\..\CodexPluginPrebuild" />
+  </Target>
+
 </Project>

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -17,7 +17,7 @@ namespace CodexPlugin
         ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration);
         ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks);
         ICodexSetup EnableMetrics();
-        ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens, bool isValidator = false);
+        ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Action<IMarketplaceSetup> marketplaceSetup);
         /// <summary>
         /// Provides an invalid proof every N proofs
         /// </summary>
@@ -25,8 +25,23 @@ namespace CodexPlugin
         ICodexSetup AsPublicTestNet(CodexTestNetConfig testNetConfig);
     }
 
+    public interface IMarketplaceSetup
+    {
+        IMarketplaceSetup WithInitial(Ether eth, TestToken tokens);
+        IMarketplaceSetup WithAccount(EthAccount account);
+        IMarketplaceSetup AsStorageNode();
+        IMarketplaceSetup AsValidator();
+    }
+
     public class CodexLogCustomTopics
     {
+        public CodexLogCustomTopics(CodexLogLevel discV5, CodexLogLevel libp2p, CodexLogLevel blockExchange)
+        {
+            DiscV5 = discV5;
+            Libp2p = libp2p;
+            BlockExchange = blockExchange;
+        }
+
         public CodexLogCustomTopics(CodexLogLevel discV5, CodexLogLevel libp2p)
         {
             DiscV5 = discV5;
@@ -35,6 +50,7 @@ namespace CodexPlugin
 
         public CodexLogLevel DiscV5 { get; set; }
         public CodexLogLevel Libp2p { get; set; }
+        public CodexLogLevel? BlockExchange { get; }
     }
 
     public class CodexSetup : CodexStartupConfig, ICodexSetup
@@ -60,7 +76,7 @@ namespace CodexPlugin
 
         public ICodexSetup WithBootstrapNode(ICodexNode node)
         {
-            BootstrapSpr = node.GetDebugInfo().spr;
+            BootstrapSpr = node.GetDebugInfo().Spr;
             return this;
         }
 
@@ -107,9 +123,12 @@ namespace CodexPlugin
             return this;
         }
 
-        public ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens, bool isValidator = false)
+        public ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Action<IMarketplaceSetup> marketplaceSetup)
         {
-            MarketplaceConfig = new MarketplaceInitialConfig(gethNode, codexContracts, initialEth, initialTokens, isValidator);
+            var ms = new MarketplaceSetup();
+            marketplaceSetup(ms);
+
+            MarketplaceConfig = new MarketplaceInitialConfig(ms, gethNode, codexContracts);
             return this;
         }
 
@@ -133,12 +152,57 @@ namespace CodexPlugin
 
         private IEnumerable<string> DescribeArgs()
         {
-            if (PublicTestNet != null) yield return $"<!>Public TestNet at {PublicTestNet.PublicNatIP}:{PublicTestNet.PublicListenPort}<!>";
+            if (PublicTestNet != null) yield return $"<!>Public TestNet with listenPort: {PublicTestNet.PublicListenPort}<!>";
             yield return $"LogLevel={LogLevelWithTopics()}";
             if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
             if (StorageQuota != null) yield return $"StorageQuota={StorageQuota}";
             if (SimulateProofFailures != null) yield return $"SimulateProofFailures={SimulateProofFailures}";
-            if (MarketplaceConfig != null) yield return $"IsValidator={MarketplaceConfig.IsValidator}";
+            if (MarketplaceConfig != null) yield return $"MarketplaceSetup={MarketplaceConfig.MarketplaceSetup}";
+        }
+    }
+
+    public class MarketplaceSetup : IMarketplaceSetup
+    {
+        public bool IsStorageNode { get; private set; }
+        public bool IsValidator { get; private set; }
+        public Ether InitialEth { get; private set; } = 0.Eth();
+        public TestToken InitialTestTokens { get; private set; } = 0.TestTokens();
+        public EthAccount EthAccount { get; private set; } = EthAccount.GenerateNew();
+
+        public IMarketplaceSetup AsStorageNode()
+        {
+            IsStorageNode = true;
+            return this;
+        }
+
+        public IMarketplaceSetup AsValidator()
+        {
+            IsValidator = true;
+            return this;
+        }
+
+        public IMarketplaceSetup WithAccount(EthAccount account)
+        {
+            EthAccount = account;
+            return this;
+        }
+
+        public IMarketplaceSetup WithInitial(Ether eth, TestToken tokens)
+        {
+            InitialEth = eth;
+            InitialTestTokens = tokens;
+            return this;
+        }
+
+        public override string ToString()
+        {
+            var result = "[(clientNode)"; // When marketplace is enabled, being a clientNode is implicit.
+            result += IsStorageNode ? "(storageNode)" : "()";
+            result += IsValidator ? "(validator)" : "() ";
+            result += $"Address: '{EthAccount.EthAddress}' ";
+            result += $"InitialEth/TT({InitialEth.Eth}/{InitialTestTokens.Amount})";
+            result += "] ";
+            return result;
         }
     }
 }

ProjectPlugins/CodexPlugin/CodexStarter.cs

@@ -1,5 +1,6 @@
 using Core;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 
 namespace CodexPlugin
@@ -8,11 +9,14 @@ namespace CodexPlugin
     {
         private readonly IPluginTools pluginTools;
         private readonly CodexContainerRecipe recipe = new CodexContainerRecipe();
-        private CodexDebugVersionResponse? versionResponse;
+        private readonly ApiChecker apiChecker;
+        private DebugInfoVersion? versionResponse;
 
         public CodexStarter(IPluginTools pluginTools)
         {
             this.pluginTools = pluginTools;
+
+            apiChecker = new ApiChecker(pluginTools);
         }
 
         public RunningContainers[] BringOnline(CodexSetup codexSetup)
@@ -24,8 +28,14 @@ namespace CodexPlugin
 
             var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
 
-            var podInfos = string.Join(", ", containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
-            Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers().First().Recipe.Image}'. ({podInfos})");
+            apiChecker.CheckCompatibility(containers);
+
+            foreach (var rc in containers)
+            {
+                var podInfo = GetPodInfo(rc);
+                var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
+                Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers().First().Recipe.Image}'. ({podInfos})");
+            }
             LogSeparator();
 
             return containers;
@@ -43,24 +53,30 @@ namespace CodexPlugin
             return group;
         }
 
-        public void BringOffline(CodexNodeGroup group)
+        public void BringOffline(CodexNodeGroup group, bool waitTillStopped)
         {
             Log($"Stopping {group.Describe()}...");
             StopCrashWatcher(group);
             var workflow = pluginTools.CreateWorkflow();
             foreach (var c in group.Containers)
             {
-                workflow.Stop(c);
+                workflow.Stop(c, waitTillStopped);
             }
             Log("Stopped.");
         }
 
         public string GetCodexId()
         {
-            if (versionResponse != null) return versionResponse.version;
+            if (versionResponse != null) return versionResponse.Version;
             return recipe.Image;
         }
 
+        public string GetCodexRevision()
+        {
+            if (versionResponse != null) return versionResponse.Revision;
+            return "unknown";
+        }
+
         private StartupConfig CreateStartupConfig(CodexSetup codexSetup)
         {
             var startupConfig = new StartupConfig();
@@ -80,6 +96,12 @@ namespace CodexPlugin
             return result.ToArray();
         }
 
+        private PodInfo GetPodInfo(RunningContainers rc)
+        {
+            var workflow = pluginTools.CreateWorkflow();
+            return workflow.GetPodInfo(rc);
+        }
+
         private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory);

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -52,12 +52,38 @@ namespace CodexPlugin
                     "connection",
                     "connmanager",
                     "websock",
-                    "ws-session"
+                    "ws-session",
+                    "dialer",
+                    "muxedupgrade",
+                    "upgrade",
+                    "identify"
+                };
+                var blockExchangeTopics = new[]
+                {
+                    "codex",
+                    "pendingblocks",
+                    "peerctxstore",
+                    "discoveryengine",
+                    "blockexcengine",
+                    "blockexcnetwork",
+                    "blockexcnetworkpeer"
+                };
+                var contractClockTopics = new[]
+                {
+                    "contracts",
+                    "clock"
                 };
 
                 level = $"{level};" +
                     $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" +
-                    $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)}";
+                    $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)};" +
+                    // Contract clock is always set to warn. It logs a trace every second.
+                    $"{CodexLogLevel.Warn.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)}";
+
+                if (CustomTopics.BlockExchange != null)
+                {
+                    level += $";{CustomTopics.BlockExchange.ToString()!.ToLowerInvariant()}:{string.Join(",", blockExchangeTopics)}";
+                }
             }
             return level;
         }
@@ -65,7 +91,6 @@ namespace CodexPlugin
 
     public class CodexTestNetConfig
     {
-        public string PublicNatIP { get; set; } = string.Empty;
         public int PublicDiscoveryPort { get; set; }
         public int PublicListenPort { get; set; }
     }