POC: codex commit blamer

This PR removes the notion of a run id and replaces it with a deploy id in continuous tests. Deploy ids can be set at deploy time only (duh), and will be picked up by the test runner from the deploy file on subsequent runs of the continuous test runner. As a consequence, the deploy id becomes a deployer parameter, and can no longer be overridden at the runner. For non-continuous tests, the deploy ID is created on-the-fly.

.github/workflows/dist-tests.yaml

@@ -1,81 +0,0 @@
-name: Dist Tests
-
-
-on:
-  # push:
-  #   branches:
-  #     - master
-  #   tags:
-  #     - 'v*.*.*'
-  #   paths-ignore:
-  #     - '**/*.md'
-  #     - '.gitignore'
-  #     - 'docker/**'
-  #     - '!docker/job.yaml'
-  workflow_dispatch:
-    inputs:
-      branch:
-        description: Branch (master)
-        required: false
-        type: string
-      source:
-        description: Repository with tests (current)
-        required: false
-        type: string
-      nameprefix:
-        description: Runner prefix (codex-dist-tests)
-        required: false
-        type: string
-      namespace:
-        description: Runner namespace (default)
-        required: false
-        type: string
-      command:
-        description: Runner command (dotnet test Tests)
-        required: false
-        type: string
-
-
-env:
-  BRANCH: ${{ github.ref_name }}
-  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
-  NAMEPREFIX: codex-dist-tests
-  NAMESPACE: default
-  COMMAND: dotnet test Tests/CodexTests
-  JOB_MANIFEST: docker/job.yaml
-  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
-  KUBE_VERSION: v1.26.1
-
-
-jobs:
-  run_tests:
-    name: Run Tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Variables
-        run: |
-          [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.command }}" ]] && COMMAND="${{ github.event.inputs.command }}" || COMMAND="${{ env.COMMAND }}"
-          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'$COMMAND'"')" >>"$GITHUB_ENV"
-          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
-          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
-
-      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
-        uses: azure/setup-kubectl@v3
-        with:
-          version: ${{ env.KUBE_VERSION }}
-
-      - name: Kubectl - Kubeconfig
-        run: |
-          mkdir -p "${HOME}"/.kube
-          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
-
-      - name: Kubectl - Create Job
-        run: |
-          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -

.github/workflows/docker-discordbot.yml

@@ -0,0 +1,28 @@
+name: Docker - Discord Bot
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/BiblioTech/**'
+      - '!Tools/BiblioTech/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-discordbot.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/BiblioTech/docker/Dockerfile
+      docker_repo: codexstorage/codex-discordbot
+    secrets: inherit
+

.github/workflows/docker-reusable.yml

@@ -1,4 +1,4 @@
-name: Docker - Reusable
+name: Reusable - Docker
 
 
 on:
@@ -66,26 +66,26 @@ jobs:
       PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Docker - Meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.DOCKER_REPO }}
 
       - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Docker - Build and Push by digest
         id: build
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ${{ env.DOCKER_FILE }}
@@ -101,9 +101,9 @@ jobs:
           touch "/tmp/digests/${digest#sha256:}"
 
       - name: Docker - Upload digest
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: digests
+          name: digests-${{ matrix.target.arch }}
           path: /tmp/digests/*
           if-no-files-found: error
           retention-days: 1
@@ -139,17 +139,18 @@ jobs:
           fi
 
       - name: Docker - Download digests
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: digests
+          pattern: digests-*
+          merge-multiple: true
           path: /tmp/digests
 
       - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Docker - Meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.DOCKER_REPO }}
           flavor: |
@@ -161,7 +162,7 @@ jobs:
             type=sha,enable=${{ env.TAG_SHA }}
 
       - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/docker-rewarder.yml

@@ -0,0 +1,26 @@
+name: Docker - Rewarder Bot
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/TestNetRewarder/**'
+      - '!Tools/TestNetRewarder/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-rewarder.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/TestNetRewarder/docker/Dockerfile
+      docker_repo: codexstorage/codex-rewarderbot
+    secrets: inherit
+

.github/workflows/docker-runner.yml

@@ -1,4 +1,4 @@
-name: Docker
+name: Docker - Runner
 
 
 on:
@@ -10,7 +10,7 @@ on:
     paths:
       - docker/Dockerfile
       - docker/docker-entrypoint.sh
-      - .github/workflows/docker.yml
+      - .github/workflows/docker-runner.yml
       - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 

.github/workflows/report-HoldMyBeerTest.yaml

@@ -0,0 +1,20 @@
+name: Report - HoldMyBeerTest
+
+
+on:
+  # schedule:
+  #   - cron: '30 1 */2 * *'
+  workflow_dispatch:
+
+jobs:
+  run_tests:
+    name: Call runner
+    uses: ./.github/workflows/run-continuous-tests.yaml
+    with:
+      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+      branch: master
+      nameprefix: c-tests-report-holdmybeertest-48h
+      tests_target_duration: 48h
+      tests_filter: HoldMyBeerTest
+      tests_cleanup: true
+    secrets: inherit

.github/workflows/report-PeersTest.yaml

@@ -0,0 +1,20 @@
+name: Report - PeersTest
+
+
+on:
+  # schedule:
+  #   - cron: '30 1 */2 * *'
+  workflow_dispatch:
+
+jobs:
+  run_tests:
+    name: Call runner
+    uses: ./.github/workflows/run-continuous-tests.yaml
+    with:
+      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+      branch: master
+      nameprefix: c-tests-report-peerstest-48h
+      tests_target_duration: 48h
+      tests_filter: PeersTest
+      tests_cleanup: true
+    secrets: inherit

.github/workflows/run-continuous-tests.yaml

@@ -0,0 +1,170 @@
+name: Run Continuous Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
+        required: false
+        type: string
+      nameprefix:
+        description: Resources prefix (c-tests)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (2d)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup
+        required: false
+        type: boolean
+        default: true
+  workflow_call:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
+        required: false
+        type: string
+      nameprefix:
+        description: Resources prefix (c-tests)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (2d)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup
+        required: false
+        type: boolean
+        default: true
+      workflow_source:
+        description: Workflow source
+        required: false
+        type: string
+        default: ''
+
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
+  NAMEPREFIX: c-tests
+  NAMESPACE: default
+  TESTS_TARGET_DURATION: 2d
+  TESTS_FILTER: ""
+  TESTS_CLEANUP: true
+  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.28.2
+
+
+jobs:
+  run_tests:
+    name: Run Continuous Tests ${{ inputs.tests_filter }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
+          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          echo "DEPLOYMENT_NAMESPACE=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@releases/v4.0.0
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}"
+          echo "Runner pod: `kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}'`"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+          echo "Tests runid: ${{ env.RUNID }}"
+          echo "Tests codexdockerimage: ${{ env.CODEXDOCKERIMAGE }}"
+          echo "Tests namespace: ${{ env.DEPLOYMENT_NAMESPACE }}"
+          echo "Tests duration: ${{ env.TESTS_TARGET_DURATION }}"
+          echo "Tests filter: ${{ env.TESTS_FILTER }}"
+          echo "Tests cleanup: ${{ env.TESTS_CLEANUP }}"
+          echo "----"
+
+      - name: Show Runner logs
+        if: false
+        run: |
+          # Variables
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          duration=600
+          namespace="${{ env.NAMESPACE }}"
+          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")
+
+          # Check Pod status
+          WAIT=120
+          SECONDS=0
+          sleep=1
+          while (( SECONDS < WAIT )); do
+            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath="{.status.phase}")
+            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in $phase state - Get the logs"; break; } || { echo "Pod $pod is in $phase state - Retry in $sleep seconds / $((WAIT - SECONDS))"; }
+            sleep $sleep
+          done
+
+          # Get logs
+          timeout $duration \
+            kubectl logs $pod \
+              -n $namespace \
+              -f \
+              --tail=-1 \
+              --timestamps || true

.github/workflows/run-dist-tests.yaml

@@ -0,0 +1,83 @@
+name: Run Dist Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      nameprefix:
+        description: Runner name prefix (d-tests-runner)
+        required: false
+        type: string
+      namespace:
+        description: Runner namespace (default)
+        required: false
+        type: string
+      command:
+        description: Command (dotnet test Tests/CodexTests)
+        required: false
+        type: string
+
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  NAMEPREFIX: d-tests-runner
+  NAMESPACE: default
+  COMMAND: dotnet test Tests/CodexTests
+  JOB_MANIFEST: docker/dist-tests-job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.28.2
+
+
+jobs:
+  run_tests:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Variables
+        run: |
+          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'$COMMAND'"')" >>"$GITHUB_ENV"
+          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@releases/v4.0.0
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+          echo "Tests runid: ${{ env.RUNID }}"
+          echo "Tests command: `jq -r '. | join(" ")' <<< '${{ env.COMMAND }}'`"
+          echo "----"

.gitignore

@@ -1,4 +1,5 @@
 .vs
 obj
 bin
-.vscode
+.vscode
+Tools/CommitBlamer/CodexTestLogs

Framework/Core/CoreInterface.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace Core
 {

Framework/Core/Http.cs

@@ -13,15 +13,15 @@ namespace Core
         T HttpGetJson<T>(string route);
         TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
         string HttpPostJson<TRequest>(string route, TRequest body);
-        string HttpPostString(string route, string body);
         TResponse HttpPostString<TResponse>(string route, string body);
         string HttpPostStream(string route, Stream stream);
         Stream HttpGetStream(string route);
-        T TryJsonDeserialize<T>(string json);
+        T Deserialize<T>(string json);
     }
 
     internal class Http : IHttp
     {
+        private static readonly object httpLock = new object();
         private readonly ILog log;
         private readonly ITimeSet timeSet;
         private readonly Address address;
@@ -48,70 +48,60 @@ namespace Core
 
         public string HttpGetString(string route)
         {
-            return Retry(() =>
+            return LockRetry(() =>
             {
-                using var client = GetClient();
+                return GetString(route);
-                var url = GetUrl() + route;
-                Log(url, "");
-                var result = Time.Wait(client.GetAsync(url));
-                var str = Time.Wait(result.Content.ReadAsStringAsync());
-                Log(url, str);
-                return str; ;
             }, $"HTTP-GET:{route}");
         }
 
         public T HttpGetJson<T>(string route)
         {
-            var json = HttpGetString(route);
+            return LockRetry(() =>
-            return TryJsonDeserialize<T>(json);
+            {
+                var json = GetString(route);
+                return Deserialize<T>(json);
+            }, $"HTTP-GET:{route}");
         }
 
         public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
         {
-            var response = PostJson(route, body);
+            return LockRetry(() =>
-            var json = Time.Wait(response.Content.ReadAsStringAsync());
-            if (!response.IsSuccessStatusCode)
             {
-                throw new HttpRequestException(json);
+                var response = PostJson(route, body);
-            }
+                var json = Time.Wait(response.Content.ReadAsStringAsync());
-            Log(GetUrl() + route, json);
+                if (!response.IsSuccessStatusCode)
-            return TryJsonDeserialize<TResponse>(json);
+                {
+                    throw new HttpRequestException(json);
+                }
+                Log(GetUrl() + route, json);
+                return Deserialize<TResponse>(json);
+            }, $"HTTP-POST-JSON: {route}");
         }
 
         public string HttpPostJson<TRequest>(string route, TRequest body)
         {
-            var response = PostJson(route, body);
+            return LockRetry(() =>
-            return Time.Wait(response.Content.ReadAsStringAsync());
-        }
-
-        public string HttpPostString(string route, string body)
-        {
-            return Retry(() =>
             {
-                using var client = GetClient();
+                var response = PostJson(route, body);
-                var url = GetUrl() + route;
+                return Time.Wait(response.Content.ReadAsStringAsync());
-                Log(url, body);
+            }, $"HTTP-POST-JSON: {route}");
-                var content = new StringContent(body);
-                content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
-                var result = Time.Wait(client.PostAsync(url, content));
-                var str = Time.Wait(result.Content.ReadAsStringAsync());
-                Log(url, str);
-                return str;
-            }, $"HTTP-POST-STRING: {route}");
         }
 
         public TResponse HttpPostString<TResponse>(string route, string body)
         {
-            var response = HttpPostString(route, body);
+            return LockRetry(() =>
-            if (response == null) throw new Exception("Received no response.");
+            {
-            var result = JsonConvert.DeserializeObject<TResponse>(response);
+                var response = PostJsonString(route, body);
-            if (result == null) throw new Exception("Failed to deserialize response");
+                if (response == null) throw new Exception("Received no response.");
-            return result;
+                var result = Deserialize<TResponse>(response);
+                if (result == null) throw new Exception("Failed to deserialize response");
+                return result;
+            }, $"HTTO-POST-JSON: {route}");
         }
 
         public string HttpPostStream(string route, Stream stream)
         {
-            return Retry(() =>
+            return LockRetry(() =>
             {
                 using var client = GetClient();
                 var url = GetUrl() + route;
@@ -127,7 +117,7 @@ namespace Core
 
         public Stream HttpGetStream(string route)
         {
-            return Retry(() =>
+            return LockRetry(() =>
             {
                 var client = GetClient();
                 var url = GetUrl() + route;
@@ -136,7 +126,7 @@ namespace Core
             }, $"HTTP-GET-STREAM: {route}");
         }
 
-        public T TryJsonDeserialize<T>(string json)
+        public T Deserialize<T>(string json)
         {
             var errors = new List<string>();
             var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
@@ -154,7 +144,7 @@ namespace Core
                     }
                 }
             });
-            if (errors.Count() > 0)
+            if (errors.Count > 0)
             {
                 throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
             }
@@ -165,16 +155,37 @@ namespace Core
             return deserialized;
         }
 
+        private string GetString(string route)
+        {
+            using var client = GetClient();
+            var url = GetUrl() + route;
+            Log(url, "");
+            var result = Time.Wait(client.GetAsync(url));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
+        }
+
         private HttpResponseMessage PostJson<TRequest>(string route, TRequest body)
         {
-            return Retry(() =>
+            using var client = GetClient();
-            {
+            var url = GetUrl() + route;
-                using var client = GetClient();
+            using var content = JsonContent.Create(body);
-                var url = GetUrl() + route;
+            Log(url, JsonConvert.SerializeObject(body));
-                using var content = JsonContent.Create(body);
+            return Time.Wait(client.PostAsync(url, content));
-                Log(url, JsonConvert.SerializeObject(body));
+        }
-                return Time.Wait(client.PostAsync(url, content));
+
-            }, $"HTTP-POST-JSON: {route}");
+        private string PostJsonString(string route, string body)
+        {
+            using var client = GetClient();
+            var url = GetUrl() + route;
+            Log(url, body);
+            var content = new StringContent(body);
+            content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+            var result = Time.Wait(client.PostAsync(url, content));
+            var str = Time.Wait(result.Content.ReadAsStringAsync());
+            Log(url, str);
+            return str;
         }
 
         private string GetUrl()
@@ -194,9 +205,12 @@ namespace Core
             }
         }
 
-        private T Retry<T>(Func<T> operation, string description)
+        private T LockRetry<T>(Func<T> operation, string description)
         {
-            return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
+            lock (httpLock)
+            {
+                return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description);
+            }
         }
 
         private HttpClient GetClient()

Framework/Core/PluginTools.cs

@@ -23,6 +23,7 @@ namespace Core
     public interface IHttpFactoryTool
     {
         IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null);
+        IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet timeSet, string? logAlias = null);
         IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null);
     }
 
@@ -53,7 +54,12 @@ namespace Core
 
         public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
         {
-            return new Http(log, timeSet, address, baseUrl, onClientCreated, logAlias);
+            return CreateHttp(address, baseUrl, onClientCreated, timeSet, logAlias);
+        }
+
+        public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet ts, string? logAlias = null)
+        {
+            return new Http(log, ts, address, baseUrl, onClientCreated, logAlias);
         }
 
         public IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null)

Framework/Core/SerializeGate.cs

@@ -5,11 +5,9 @@ namespace Core
     public static class SerializeGate
     {
         /// <summary>
-        /// SerializeGate was added to help ensure deployment objects are serializable
+        /// SerializeGate was added to help ensure deployment objects are serializable and remain viable after deserialization.
-        /// and remain viable after deserialization.
         /// Tools can be built on top of the core interface that rely on deployment objects being serializable.
-        /// Insert the serialization gate after deployment but before wrapping to ensure any future changes
+        /// Insert the serialization gate after deployment but before wrapping to ensure any future changes don't break this requirement.
-        /// don't break this requirement.
         /// </summary>
         public static T Gate<T>(T anything)
         {

Framework/Core/TimeSet.cs

@@ -3,7 +3,7 @@
     public interface ITimeSet
     {
         TimeSpan HttpCallTimeout();
-        TimeSpan HttpCallRetryTime();
+        int HttpMaxNumberOfRetries();
         TimeSpan HttpCallRetryDelay();
         TimeSpan WaitForK8sServiceDelay();
         TimeSpan K8sOperationTimeout();
@@ -13,12 +13,12 @@
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromMinutes(5);
+            return TimeSpan.FromMinutes(3);
         }
 
-        public TimeSpan HttpCallRetryTime()
+        public int HttpMaxNumberOfRetries()
         {
-            return TimeSpan.FromMinutes(1);
+            return 3;
         }
 
         public TimeSpan HttpCallRetryDelay()
@@ -36,4 +36,32 @@
             return TimeSpan.FromMinutes(30);
         }
     }
+
+    public class LongTimeSet : ITimeSet
+    {
+        public TimeSpan HttpCallTimeout()
+        {
+            return TimeSpan.FromHours(2);
+        }
+
+        public int HttpMaxNumberOfRetries()
+        {
+            return 1;
+        }
+
+        public TimeSpan HttpCallRetryDelay()
+        {
+            return TimeSpan.FromSeconds(2);
+        }
+
+        public TimeSpan WaitForK8sServiceDelay()
+        {
+            return TimeSpan.FromSeconds(10);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromMinutes(15);
+        }
+    }
 }

Framework/DiscordRewards/CheckConfig.cs

@@ -0,0 +1,21 @@
+using Utils;
+
+namespace DiscordRewards
+{
+    public class CheckConfig
+    {
+        public CheckType Type { get; set; }
+        public ulong MinNumberOfHosts { get; set; }
+        public ByteSize MinSlotSize { get; set; } = 0.Bytes();
+        public TimeSpan MinDuration { get; set; } = TimeSpan.Zero;
+    }
+
+    public enum CheckType
+    {
+        Uninitialized,
+        FilledSlot,
+        FinishedSlot,
+        PostedContract,
+        StartedContract,
+    }
+}

Framework/DiscordRewards/DiscordRewards.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -0,0 +1,13 @@
+namespace DiscordRewards
+{
+    public class GiveRewardsCommand
+    {
+        public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
+    }
+
+    public class RewardUsersCommand
+    {
+        public ulong RewardId { get; set; }
+        public string[] UserAddresses { get; set; } = Array.Empty<string>();
+    }
+}

Framework/DiscordRewards/RewardConfig.cs

@@ -0,0 +1,18 @@
+namespace DiscordRewards
+{
+    public class RewardConfig
+    {
+        public const string UsernameTag = "<USER>";
+
+        public RewardConfig(ulong roleId, string message, CheckConfig checkConfig)
+        {
+            RoleId = roleId;
+            Message = message;
+            CheckConfig = checkConfig;
+        }
+
+        public ulong RoleId { get; }
+        public string Message { get; }
+        public CheckConfig CheckConfig { get; }
+    }
+}

Framework/DiscordRewards/RewardRepo.cs

@@ -0,0 +1,53 @@
+using Utils;
+
+namespace DiscordRewards
+{
+    public class RewardRepo
+    {
+        private static string Tag => RewardConfig.UsernameTag;
+
+        public RewardConfig[] Rewards { get; } = new RewardConfig[]
+        {
+            // Filled any slot
+            new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
+            {
+                Type = CheckType.FilledSlot
+            }),
+
+            // Finished any slot
+            new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
+            {
+                Type = CheckType.FinishedSlot
+            }),
+
+            // Finished a sizable slot
+            new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot!", new CheckConfig
+            {
+                Type = CheckType.FinishedSlot,
+                MinSlotSize = 1.GB(),
+                MinDuration = TimeSpan.FromHours(24.0),
+            }),
+
+            // Posted any contract
+            new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
+            {
+                Type = CheckType.PostedContract
+            }),
+
+            // Started any contract
+            new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
+            {
+                Type = CheckType.StartedContract
+            }),
+
+            // Started a sizable contract
+            new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time!", new CheckConfig
+            {
+                Type = CheckType.FinishedSlot,
+                MinNumberOfHosts = 4,
+                MinSlotSize = 1.GB(),
+                MinDuration = TimeSpan.FromHours(24.0),
+            })
+        };
+    }
+}

Framework/FileUtils/TrackedFile.cs

@@ -37,6 +37,11 @@ namespace FileUtils
             return $"'{Filename}'{sizePostfix}";
         }
 
+        public ByteSize GetFilesize()
+        {
+            return new ByteSize(GetFileSize());
+        }
+
         private void AssertEqual(TrackedFile? actual)
         {
             if (actual == null)  FrameworkAssert.Fail("TestFile is null.");

Framework/GethConnector/GethConnector.cs

@@ -0,0 +1,39 @@
+using CodexContractsPlugin;
+using GethPlugin;
+using Logging;
+
+namespace GethConnector
+{
+    public class GethConnector
+    {
+        public IGethNode GethNode { get; }
+        public ICodexContracts CodexContracts { get; }
+
+        public static GethConnector? Initialize(ILog log)
+        {
+            if (!string.IsNullOrEmpty(GethInput.LoadError))
+            {
+                var msg = "Geth input incorrect: " + GethInput.LoadError;
+                log.Error(msg);
+                return null;
+            }
+
+            var contractsDeployment = new CodexContractsDeployment(
+                marketplaceAddress: GethInput.MarketplaceAddress,
+                abi: GethInput.ABI,
+                tokenAddress: GethInput.TokenAddress
+            );
+
+            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+            var contracts = new CodexContractsAccess(log, gethNode, contractsDeployment);
+
+            return new GethConnector(gethNode, contracts);
+        }
+
+        private GethConnector(IGethNode gethNode, ICodexContracts codexContracts)
+        {
+            GethNode = gethNode;
+            CodexContracts = codexContracts;
+        }
+    }
+}

Framework/GethConnector/GethConnector.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\GethPlugin\GethPlugin.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/GethConnector/GethInput.cs

@@ -0,0 +1,52 @@
+namespace GethConnector
+{
+    public static class GethInput
+    {
+        private const string GethHostVar = "GETH_HOST";
+        private const string GethPortVar = "GETH_HTTP_PORT";
+        private const string GethPrivKeyVar = "GETH_PRIVATE_KEY";
+        private const string MarketplaceAddressVar = "CODEXCONTRACTS_MARKETPLACEADDRESS";
+        private const string TokenAddressVar = "CODEXCONTRACTS_TOKENADDRESS";
+        private const string AbiVar = "CODEXCONTRACTS_ABI";
+
+        static GethInput()
+        {
+            var error = new List<string>();
+            var gethHost = GetEnvVar(error, GethHostVar);
+            var gethPort = Convert.ToInt32(GetEnvVar(error, GethPortVar));
+            var privateKey = GetEnvVar(error, GethPrivKeyVar);
+            var marketplaceAddress = GetEnvVar(error, MarketplaceAddressVar);
+            var tokenAddress = GetEnvVar(error, TokenAddressVar);
+            var abi = GetEnvVar(error, AbiVar);
+
+            if (error.Any())
+            {
+                LoadError = string.Join(", ", error);
+            }
+            else
+            {
+                GethHost = gethHost!;
+                GethPort = gethPort;
+                PrivateKey = privateKey!;
+                MarketplaceAddress = marketplaceAddress!;
+                TokenAddress = tokenAddress!;
+                ABI = abi!;
+            }
+        }
+
+        public static string GethHost { get; } = string.Empty;
+        public static int GethPort { get; }
+        public static string PrivateKey { get; } = string.Empty;
+        public static string MarketplaceAddress { get; } = string.Empty;
+        public static string TokenAddress { get; } = string.Empty;
+        public static string ABI { get; } = string.Empty;
+        public static string LoadError { get; } = string.Empty;
+
+        private static string? GetEnvVar(List<string> error, string name)
+        {
+            var result = Environment.GetEnvironmentVariable(name);
+            if (string.IsNullOrEmpty(result)) error.Add($"'{name}' is not set.");
+            return result;
+        }
+    }
+}

Framework/KubernetesWorkflow/CommandRunner.cs

@@ -6,17 +6,17 @@ namespace KubernetesWorkflow
     {
         private readonly K8sClient client;
         private readonly string k8sNamespace;
-        private readonly RunningPod pod;
+        private readonly string podName;
         private readonly string containerName;
         private readonly string command;
         private readonly string[] arguments;
         private readonly List<string> lines = new List<string>();
 
-        public CommandRunner(K8sClient client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
+        public CommandRunner(K8sClient client, string k8sNamespace, string podName, string containerName, string command, string[] arguments)
         {
             this.client = client;
             this.k8sNamespace = k8sNamespace;
-            this.pod = pod;
+            this.podName = podName;
             this.containerName = containerName;
             this.command = command;
             this.arguments = arguments;
@@ -27,7 +27,7 @@ namespace KubernetesWorkflow
             var input = new[] { command }.Concat(arguments).ToArray();
 
             Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
-                pod.PodInfo.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
+                podName, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
         }
 
         public string GetStdOut()

Framework/KubernetesWorkflow/CrashWatcher.cs

@@ -7,19 +7,23 @@ namespace KubernetesWorkflow
     {
         private readonly ILog log;
         private readonly KubernetesClientConfiguration config;
+        private readonly string containerName;
+        private readonly string podName;
+        private readonly string recipeName;
         private readonly string k8sNamespace;
-        private readonly RunningContainer container;
         private ILogHandler? logHandler;
         private CancellationTokenSource cts;
         private Task? worker;
         private Exception? workerException;
 
-        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container)
+        public CrashWatcher(ILog log, KubernetesClientConfiguration config, string containerName, string podName, string recipeName, string k8sNamespace)
         {
             this.log = log;
             this.config = config;
+            this.containerName = containerName;
+            this.podName = podName;
+            this.recipeName = recipeName;
             this.k8sNamespace = k8sNamespace;
-            this.container = container;
             cts = new CancellationTokenSource();
         }
 
@@ -46,7 +50,7 @@ namespace KubernetesWorkflow
         public bool HasContainerCrashed()
         {
             using var client = new Kubernetes(config);
-            return HasContainerBeenRestarted(client, container.Pod.PodInfo.Name);
+            return HasContainerBeenRestarted(client);
         }
 
         private void Worker()
@@ -66,29 +70,26 @@ namespace KubernetesWorkflow
             using var client = new Kubernetes(config);
             while (!token.IsCancellationRequested)
             {
-                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(1));
+                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(10));
 
-                var pod = container.Pod;
+                if (HasContainerBeenRestarted(client))
-                var recipe = container.Recipe;
-                var podName = pod.PodInfo.Name;
-                if (HasContainerBeenRestarted(client, podName))
                 {
-                    DownloadCrashedContainerLogs(client, podName, recipe);
+                    DownloadCrashedContainerLogs(client);
                     return;
                 }
             }
         }
 
-        private bool HasContainerBeenRestarted(Kubernetes client, string podName)
+        private bool HasContainerBeenRestarted(Kubernetes client)
         {
             var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
             return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
         }
 
-        private void DownloadCrashedContainerLogs(Kubernetes client, string podName, ContainerRecipe recipe)
+        private void DownloadCrashedContainerLogs(Kubernetes client)
         {
-            log.Log("Pod crash detected for " + container.Name);
+            log.Log("Pod crash detected for " + containerName);
-            using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipe.Name, previous: true);
+            using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipeName, previous: true);
             logHandler!.Log(stream);
         }
     }

Framework/KubernetesWorkflow/K8sCluster.cs

@@ -54,16 +54,4 @@ namespace KubernetesWorkflow
             }
         }
     }
-
-    public class K8sNodeLabel
-    {
-        public K8sNodeLabel(string key, string value)
-        {
-            Key = key;
-            Value = value;
-        }
-
-        public string Key { get; }
-        public string Value { get; }
-    }
 }

Framework/KubernetesWorkflow/K8sController.cs

@@ -1,5 +1,7 @@
 using k8s;
 using k8s.Models;
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
 using Logging;
 using Utils;
 
@@ -9,15 +11,14 @@ namespace KubernetesWorkflow
     {
         private readonly ILog log;
         private readonly K8sCluster cluster;
-        private readonly KnownK8sPods knownPods;
         private readonly WorkflowNumberSource workflowNumberSource;
         private readonly K8sClient client;
+        public const string PodLabelKey = "pod-uuid";
 
-        public K8sController(ILog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string k8sNamespace)
+        public K8sController(ILog log, K8sCluster cluster, WorkflowNumberSource workflowNumberSource, string k8sNamespace)
         {
             this.log = log;
             this.cluster = cluster;
-            this.knownPods = knownPods;
             this.workflowNumberSource = workflowNumberSource;
             client = new K8sClient(cluster.GetK8sClientConfig());
 
@@ -29,40 +30,53 @@ namespace KubernetesWorkflow
             client.Dispose();
         }
 
-        public RunningPod BringOnline(ContainerRecipe[] containerRecipes, ILocation location)
+        public StartResult BringOnline(ContainerRecipe[] containerRecipes, ILocation location)
         {
             log.Debug();
             EnsureNamespace();
 
-            var deploymentName = CreateDeployment(containerRecipes, location);
+            var podLabel = K8sNameUtils.Format(Guid.NewGuid().ToString());
-            var (serviceName, servicePortsMap) = CreateService(containerRecipes);
+            var deployment = CreateDeployment(containerRecipes, location, podLabel);
-            var podInfo = FetchNewPod();
+            var internalService = CreateInternalService(containerRecipes);
+            var externalService = CreateExternalService(containerRecipes);
 
-            return new RunningPod(cluster, podInfo, deploymentName, serviceName, servicePortsMap.ToArray());
+            return new StartResult(cluster, containerRecipes, deployment, internalService, externalService);
         }
 
-        public void Stop(RunningPod pod)
+        public PodInfo GetPodInfo(RunningDeployment deployment)
         {
-            log.Debug();
+            var pod = GetPodForDeployment(deployment);
-            if (!string.IsNullOrEmpty(pod.ServiceName)) DeleteService(pod.ServiceName);
+            return CreatePodInfo(pod);
-            DeleteDeployment(pod.DeploymentName);
-            WaitUntilDeploymentOffline(pod.DeploymentName);
-            WaitUntilPodOffline(pod.PodInfo.Name);
         }
 
-        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler, int? tailLines)
+        public void Stop(StartResult startResult)
         {
             log.Debug();
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sNamespace, recipe.Name, tailLines: tailLines));
+            if (startResult.InternalService != null) DeleteService(startResult.InternalService);
+            if (startResult.ExternalService != null) DeleteService(startResult.ExternalService);
+            DeleteDeployment(startResult.Deployment);
+            WaitUntilPodsForDeploymentAreOffline(startResult.Deployment);
+        }
+
+        public void DownloadPodLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
+        {
+            log.Debug();
+
+            var podName = GetPodName(container);
+            var recipeName = container.Recipe.Name;
+
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(podName, K8sNamespace, recipeName, tailLines: tailLines));
             logHandler.Log(stream);
         }
 
-        public string ExecuteCommand(RunningPod pod, string containerName, string command, params string[] args)
+        public string ExecuteCommand(RunningContainer container, string command, params string[] args)
         {
+            var containerName = container.Recipe.Name;
             var cmdAndArgs = $"{containerName}: {command} ({string.Join(",", args)})";
             log.Debug(cmdAndArgs);
 
-            var runner = new CommandRunner(client, K8sNamespace, pod, containerName, command, args);
+            var podName = GetPodName(container);
+            var runner = new CommandRunner(client, K8sNamespace, podName, containerName, command, args);
             runner.Run();
             var result = runner.GetStdOut();
 
@@ -70,6 +84,31 @@ namespace KubernetesWorkflow
             return result;
         }
 
+        public int[] GetUsedExternalPorts()
+        {
+            return client.Run(c =>
+            {
+                var result = new List<int>();
+
+                var services = c.ListServiceForAllNamespaces();
+                var nodePorts = services.Items.Where(s => s.Spec.Type == "NodePort").ToArray();
+                if (!nodePorts.Any()) return result.ToArray();
+
+                foreach (var service in nodePorts)
+                {
+                    foreach (var port in service.Spec.Ports)
+                    {
+                        if (port.NodePort.HasValue)
+                        {
+                            result.Add(port.NodePort.Value);
+                        }
+                    }
+                }
+
+                return result.ToArray();
+            });
+        }
+
         public void DeleteAllNamespacesStartingWith(string prefix)
         {
             log.Debug();
@@ -81,10 +120,6 @@ namespace KubernetesWorkflow
             {
                 DeleteNamespace(ns);
             }
-            foreach (var ns in namespaces)
-            {
-                WaitUntilNamespaceDeleted(ns);
-            }
         }
 
         public void DeleteNamespace()
@@ -94,7 +129,6 @@ namespace KubernetesWorkflow
             {
                 client.Run(c => c.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0));
             }
-            WaitUntilNamespaceDeleted();
         }
 
         public void DeleteNamespace(string ns)
@@ -304,7 +338,7 @@ namespace KubernetesWorkflow
 
         #region Deployment management
 
-        private string CreateDeployment(ContainerRecipe[] containerRecipes, ILocation location)
+        private RunningDeployment CreateDeployment(ContainerRecipe[] containerRecipes, ILocation location, string podLabel)
         {
             var deploymentSpec = new V1Deployment
             {
@@ -321,11 +355,13 @@ namespace KubernetesWorkflow
                     {
                         Metadata = new V1ObjectMeta
                         {
-                            Labels = GetSelector(containerRecipes),
+                            Labels = GetSelector(containerRecipes, podLabel),
                             Annotations = GetAnnotations(containerRecipes)
                         },
                         Spec = new V1PodSpec
                         {
+                            PriorityClassName = GetPriorityClassName(containerRecipes),
+                            Affinity = CreatePodAffinity(containerRecipes),
                             NodeSelector = CreateNodeSelector(location),
                             Containers = CreateDeploymentContainers(containerRecipes),
                             Volumes = CreateVolumes(containerRecipes)
@@ -337,13 +373,14 @@ namespace KubernetesWorkflow
             client.Run(c => c.CreateNamespacedDeployment(deploymentSpec, K8sNamespace));
             WaitUntilDeploymentOnline(deploymentSpec.Metadata.Name);
 
-            return deploymentSpec.Metadata.Name;
+            var name = deploymentSpec.Metadata.Name;
+            return new RunningDeployment(name, podLabel);
         }
 
-        private void DeleteDeployment(string deploymentName)
+        private void DeleteDeployment(RunningDeployment deployment)
         {
-            client.Run(c => c.DeleteNamespacedDeployment(deploymentName, K8sNamespace));
+            client.Run(c => c.DeleteNamespacedDeployment(deployment.Name, K8sNamespace));
-            WaitUntilDeploymentOffline(deploymentName);
+            WaitUntilDeploymentOffline(deployment.Name);
         }
 
         private IDictionary<string, string> CreateNodeSelector(ILocation location)
@@ -357,17 +394,69 @@ namespace KubernetesWorkflow
             };
         }
 
+        private V1Affinity? CreatePodAffinity(ContainerRecipe[] recipes)
+        {
+            var notIns = recipes
+                .Select(r => r.SchedulingAffinity.NotIn)
+                .Where(n => !string.IsNullOrEmpty(n))
+                .Distinct()
+                .ToList();
+
+            if (!notIns.Any()) return null;
+
+            return new V1Affinity
+            {
+                NodeAffinity = new V1NodeAffinity
+                {
+                    RequiredDuringSchedulingIgnoredDuringExecution = new V1NodeSelector
+                    {
+                        NodeSelectorTerms = new List<V1NodeSelectorTerm>
+                        { 
+                            new V1NodeSelectorTerm
+                            {
+                                MatchExpressions = new List<V1NodeSelectorRequirement>
+                                {
+                                    new V1NodeSelectorRequirement
+                                    {
+                                        Key = "allow-tests-pods",
+                                        OperatorProperty = "NotIn",
+                                        Values = notIns
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            };
+        }
+
         private K8sNodeLabel? GetNodeLabelForLocation(ILocation location)
         {
             var l = (Location)location;
             return l.NodeLabel;
         }
 
+        private string GetPriorityClassName(ContainerRecipe[] containerRecipes)
+        {
+            if (containerRecipes.Any(c => c.SetCriticalPriority))
+            {
+                return "system-node-critical";
+            }
+            return null!;
+        }
+
         private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes)
         {
             return containerRecipes.First().PodLabels.GetLabels();
         }
 
+        private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes, string podLabel)
+        {
+            var labels = containerRecipes.First().PodLabels.Clone();
+            labels.Add(PodLabelKey, podLabel);
+            return labels.GetLabels();
+        }
+
         private IDictionary<string, string> GetRunnerNamespaceSelector()
         {
             return new Dictionary<string, string> { { "kubernetes.io/metadata.name", "default" } };
@@ -446,7 +535,8 @@ namespace KubernetesWorkflow
             return new V1VolumeMount
             {
                 Name = v.VolumeName,
-                MountPath = v.MountPath
+                MountPath = v.MountPath,
+                SubPath = v.SubPath,
             };
         }
 
@@ -462,28 +552,28 @@ namespace KubernetesWorkflow
 
         private V1Volume CreateVolume(VolumeMount v)
         {
-            client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim
+            CreatePersistentVolumeClaimIfNeeded(v);
+
+            if (!string.IsNullOrEmpty(v.HostPath))
             {
-                ApiVersion = "v1",
+                return new V1Volume
-                Metadata = new V1ObjectMeta
                 {
-                    Name = v.VolumeName
+                    Name = v.VolumeName,
-                },
+                    HostPath = new V1HostPathVolumeSource
-                Spec = new V1PersistentVolumeClaimSpec
-                {
-                    AccessModes = new List<string>
                     {
-                        "ReadWriteOnce"
+                        Path = v.HostPath
-                    },
-                    Resources = new V1ResourceRequirements
-                    {
-                        Requests = new Dictionary<string, ResourceQuantity>
-                        {
-                            {"storage", new ResourceQuantity(v.ResourceQuantity) }
-                        }
                     }
-                }
+                };
-            }, K8sNamespace));
+            }
+
+            if (!string.IsNullOrEmpty(v.Secret))
+            {
+                return new V1Volume
+                {
+                    Name = v.VolumeName,
+                    Secret = CreateVolumeSecret(v)
+                };
+            }
 
             return new V1Volume
             {
@@ -495,6 +585,50 @@ namespace KubernetesWorkflow
             };
         }
 
+        private void CreatePersistentVolumeClaimIfNeeded(VolumeMount v)
+        {
+            var pvcs = client.Run(c => c.ListNamespacedPersistentVolumeClaim(K8sNamespace));
+            if (pvcs != null && pvcs.Items.Any(i => i.Name() == v.VolumeName)) return;
+
+            client.Run(c => c.CreateNamespacedPersistentVolumeClaim(new V1PersistentVolumeClaim
+            {
+                ApiVersion = "v1",
+                Metadata = new V1ObjectMeta
+                {
+                    Name = v.VolumeName,
+                },
+                Spec = new V1PersistentVolumeClaimSpec
+                {
+                    AccessModes = new List<string>
+                    {
+                        "ReadWriteOnce"
+                    },
+                    Resources = CreateVolumeResourceRequirements(v),
+                },
+            }, K8sNamespace));
+        }
+
+        private V1SecretVolumeSource CreateVolumeSecret(VolumeMount v)
+        {
+            if (string.IsNullOrWhiteSpace(v.Secret)) return null!;
+            return new V1SecretVolumeSource
+            {
+                SecretName = v.Secret
+            };
+        }
+
+        private V1ResourceRequirements CreateVolumeResourceRequirements(VolumeMount v)
+        {
+            if (v.ResourceQuantity == null) return null!;
+            return new V1ResourceRequirements
+            {
+                Requests = new Dictionary<string, ResourceQuantity>()
+                {
+                    {"storage", new ResourceQuantity(v.ResourceQuantity) }
+                }
+            };
+        }
+
         private List<V1EnvVar> CreateEnv(ContainerRecipe recipe)
         {
             return recipe.EnvVars.Select(CreateEnvVar).ToList();
@@ -511,132 +645,211 @@ namespace KubernetesWorkflow
 
         private List<V1ContainerPort> CreateContainerPorts(ContainerRecipe recipe)
         {
-            var exposedPorts = recipe.ExposedPorts.Select(p => CreateContainerPort(recipe, p));
+            var exposedPorts = recipe.ExposedPorts.SelectMany(p => CreateContainerPort(recipe, p));
-            var internalPorts = recipe.InternalPorts.Select(p => CreateContainerPort(recipe, p));
+            var internalPorts = recipe.InternalPorts.SelectMany(p => CreateContainerPort(recipe, p));
             return exposedPorts.Concat(internalPorts).ToList();
         }
 
-        private V1ContainerPort CreateContainerPort(ContainerRecipe recipe, Port port)
+        private List<V1ContainerPort> CreateContainerPort(ContainerRecipe recipe, Port port)
+        {
+            var result = new List<V1ContainerPort>();
+            if (port.IsTcp()) CreateTcpContainerPort(result, recipe, port);
+            if (port.IsUdp()) CreateUdpContainerPort(result, recipe, port);
+            return result;
+        }
+
+        private void CreateUdpContainerPort(List<V1ContainerPort> result, ContainerRecipe recipe, Port port)
+        {
+            result.Add(CreateContainerPort(recipe, port, "UDP"));
+        }
+
+        private void CreateTcpContainerPort(List<V1ContainerPort> result, ContainerRecipe recipe, Port port)
+        {
+            result.Add(CreateContainerPort(recipe, port, "TCP"));
+        }
+
+        private V1ContainerPort CreateContainerPort(ContainerRecipe recipe, Port port, string protocol)
         {
             return new V1ContainerPort
             {
                 Name = GetNameForPort(recipe, port),
-                ContainerPort = port.Number
+                ContainerPort = port.Number,
+                Protocol = protocol
             };
         }
 
         private string GetNameForPort(ContainerRecipe recipe, Port port)
         {
-            return $"p{workflowNumberSource.WorkflowNumber}-{recipe.Number}-{port.Number}";
+            var inputs = new[]
+            {
+                $"p{workflowNumberSource.WorkflowNumber}",
+                recipe.Number.ToString(),
+                port.Number.ToString(),
+                port.Protocol.ToString().ToLowerInvariant()
+            };
+
+            return K8sNameUtils.FormatPortName(string.Join(",", inputs));
+        }
+
+        private string GetPodName(RunningContainer container)
+        {
+            return GetPodForDeployment(container.RunningContainers.StartResult.Deployment).Metadata.Name;
+        }
+
+        private V1Pod GetPodForDeployment(RunningDeployment deployment)
+        {
+            return Time.Retry(() => GetPodForDeplomentInternal(deployment),
+                // We will wait up to 1 minute, k8s might be moving pods around.
+                maxRetries: 6,
+                retryTime: TimeSpan.FromSeconds(10),
+                description: "Find pod by label for deployment.");
+        }
+
+        private V1Pod GetPodForDeplomentInternal(RunningDeployment deployment)
+        {
+            var allPods = client.Run(c => c.ListNamespacedPod(K8sNamespace));
+            var pods = allPods.Items.Where(p => p.GetLabel(PodLabelKey) == deployment.PodLabel).ToArray();
+
+            if (pods.Length != 1)
+            {
+                var allLabels = allPods.Items.Select(p =>
+                {
+                    var labels = string.Join(",", p.Labels().Select(l => $"{l.Key}={l.Value}"));
+                    return $"pod:'{p.Name()}' has labels: [{labels}]";
+                });
+                throw new Exception($"Expected to find 1 pod by podLabel '{deployment.PodLabel}'. Found: {pods.Length}. " +
+                    $"Total number of pods: {allPods.Items.Count}. Their labels: {string.Join(Environment.NewLine, allLabels)}");
+            }
+            return pods[0];
         }
 
         #endregion
 
         #region Service management
 
-        private (string, List<ContainerRecipePortMapEntry>) CreateService(ContainerRecipe[] containerRecipes)
+        private RunningService? CreateInternalService(ContainerRecipe[] recipes)
         {
-            var result = new List<ContainerRecipePortMapEntry>();
+            return CreateService(recipes, r => r.InternalPorts.Concat(r.ExposedPorts).ToArray(), "ClusterIP", "int", false);
+        }
 
-            var ports = CreateServicePorts(containerRecipes);
+        private RunningService? CreateExternalService(ContainerRecipe[] recipes)
+        {
+            return CreateService(recipes, r => r.ExposedPorts, "NodePort", "ext", true);
+        }
 
-            if (!ports.Any())
+        private RunningService? CreateService(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector, string serviceType, string namePostfix, bool isNodePort)
-            {
+        {
-                // None of these container-recipes wish to expose anything via a service port.
+            var ports = CreateServicePorts(recipes, portSelector, isNodePort);
-                // So, we don't have to create a service.
+            if (!ports.Any()) return null;
-                return (string.Empty, result);
-            }
 
             var serviceSpec = new V1Service
             {
                 ApiVersion = "v1",
-                Metadata = CreateServiceMetadata(),
+                Metadata = CreateServiceMetadata(recipes, namePostfix),
                 Spec = new V1ServiceSpec
                 {
-                    Type = "NodePort",
+                    Type = serviceType,
-                    Selector = GetSelector(containerRecipes),
+                    Selector = GetSelector(recipes),
-                    Ports = ports
+                    Ports = ports,
                 }
             };
 
             client.Run(c => c.CreateNamespacedService(serviceSpec, K8sNamespace));
 
-            ReadBackServiceAndMapPorts(serviceSpec, containerRecipes, result);
+            var result = ReadBackServiceAndMapPorts(serviceSpec, recipes);
+            var name = serviceSpec.Metadata.Name;
 
-            return (serviceSpec.Metadata.Name, result);
+            return new RunningService(name, result);
         }
 
-        private void ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes, List<ContainerRecipePortMapEntry> result)
+        private List<ContainerRecipePortMapEntry> ReadBackServiceAndMapPorts(V1Service serviceSpec, ContainerRecipe[] containerRecipes)
         {
-            // For each container-recipe, we need to figure out which service-ports it was assigned by K8s.
+            var result = new List<ContainerRecipePortMapEntry>();
+
+            // For each container-recipe-port, we need to figure out which service-ports it was assigned by K8s.
             var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sNamespace));
             foreach (var r in containerRecipes)
             {
-                if (r.ExposedPorts.Any())
+                var recipePorts = r.ExposedPorts.Concat(r.InternalPorts).ToArray();
+                foreach (var port in recipePorts)
                 {
-                    var firstExposedPort = r.ExposedPorts.First();
+                    var portName = GetNameForPort(r, port);
-                    var portName = GetNameForPort(r, firstExposedPort);
 
                     var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName);
-                    if (matchingServicePorts.Any())
+                    var ports = matchingServicePorts.Select(p => MapPortIfAble(p, port.Tag, port.Protocol)).ToArray();
-                    {
-                        // These service ports belongs to this recipe.
-                        var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
-                        var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
 
+                    if (ports.Any())
+                    {
                         result.Add(new ContainerRecipePortMapEntry(r.Number, ports));
+                        log.Debug($"Service Readback: {portName} found: {string.Join(",", ports.Select(p => p.ToString()))}");
                     }
                 }
             }
+
+            return result;
         }
 
-        private Port? MapNodePortIfAble(V1ServicePort p, string tag)
+        private Port MapPortIfAble(V1ServicePort p, string tag, PortProtocol protocol)
         {
-            if (p.NodePort == null) return null;
+            if (p.NodePort != null) return new Port(p.NodePort.Value, tag, protocol);
-            return new Port(p.NodePort.Value, tag);
+            if (p.Port > 0) return new Port(p.Port, tag, protocol);
+            throw new Exception("Unable to map port.");
         }
 
-        private void DeleteService(string serviceName)
+        private void DeleteService(RunningService service)
         {
-            client.Run(c => c.DeleteNamespacedService(serviceName, K8sNamespace));
+            client.Run(c => c.DeleteNamespacedService(service.Name, K8sNamespace));
         }
 
-        private V1ObjectMeta CreateServiceMetadata()
+        private V1ObjectMeta CreateServiceMetadata(ContainerRecipe[] containerRecipes, string namePostfix)
         {
+            var recipeName = containerRecipes.First().Name;
+            var name = K8sNameUtils.Format($"{recipeName}-{workflowNumberSource.WorkflowNumber}-{namePostfix}");
+            log.Debug("Creating service: " + name);
             return new V1ObjectMeta
             {
-                Name = "service-" + workflowNumberSource.WorkflowNumber,
+                Name = name,
-                NamespaceProperty = K8sNamespace
+                NamespaceProperty = K8sNamespace,
             };
         }
 
-        private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes)
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector, bool isNodePort)
         {
             var result = new List<V1ServicePort>();
             foreach (var recipe in recipes)
             {
-                result.AddRange(CreateServicePorts(recipe));
+                var ports = portSelector(recipe);
+                foreach (var port in ports)
+                {
+                    result.AddRange(CreateServicePorts(recipe, port, isNodePort));
+                }
             }
             return result;
         }
 
-        private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe)
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe, Port recipePort, bool isNodePort)
         {
             var result = new List<V1ServicePort>();
-            foreach (var port in recipe.ExposedPorts)
+            if (recipePort.IsTcp()) CreateServicePort(result, recipe, recipePort, "TCP", isNodePort);
-            {
+            if (recipePort.IsUdp()) CreateServicePort(result, recipe, recipePort, "UDP", isNodePort);
-                result.Add(new V1ServicePort
-                {
-                    Name = GetNameForPort(recipe, port),
-                    Protocol = "TCP",
-                    Port = port.Number,
-                    TargetPort = GetNameForPort(recipe, port),
-                });                
-            }
-
             return result;
         }
 
+        private void CreateServicePort(List<V1ServicePort> result, ContainerRecipe recipe, Port port, string protocol, bool isNodePort)
+        {
+            var p = new V1ServicePort
+            {
+                Name = GetNameForPort(recipe, port),
+                Protocol = protocol,
+                Port = port.Number,
+                TargetPort = GetNameForPort(recipe, port)
+            };
+
+            if (isNodePort) p.NodePort = port.Number;
+
+            result.Add(p);
+        }
+
         #endregion
 
         #region Waiting
@@ -646,16 +859,6 @@ namespace KubernetesWorkflow
             WaitUntil(() => IsNamespaceOnline(K8sNamespace));
         }
 
-        private void WaitUntilNamespaceDeleted()
-        {
-            WaitUntil(() => !IsNamespaceOnline(K8sNamespace));
-        }
-
-        private void WaitUntilNamespaceDeleted(string name)
-        {
-            WaitUntil(() => !IsNamespaceOnline(name));
-        }
-
         private void WaitUntilDeploymentOnline(string deploymentName)
         {
             WaitUntil(() =>
@@ -675,13 +878,12 @@ namespace KubernetesWorkflow
             });
         }
 
-        private void WaitUntilPodOffline(string podName)
+        private void WaitUntilPodsForDeploymentAreOffline(RunningDeployment deployment)
         {
             WaitUntil(() =>
             {
-                var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)).Items;
+                var pods = FindPodsByLabel(deployment.PodLabel);
-                var pod = pods.SingleOrDefault(p => p.Metadata.Name == podName);
+                return !pods.Any();
-                return pod == null;
             });
         }
 
@@ -702,25 +904,28 @@ namespace KubernetesWorkflow
 
         public CrashWatcher CreateCrashWatcher(RunningContainer container)
         {
-            return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sNamespace, container);
+            var containerName = container.Name;
+            var podName = GetPodName(container);
+            var recipeName = container.Recipe.Name;
+
+            return new CrashWatcher(log, cluster.GetK8sClientConfig(), containerName, podName, recipeName, K8sNamespace);
         }
 
-        private PodInfo FetchNewPod()
+        private V1Pod[] FindPodsByLabel(string podLabel)
         {
-            var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace)).Items;
+            var pods = client.Run(c => c.ListNamespacedPod(K8sNamespace));
+            return pods.Items.Where(p => p.GetLabel(PodLabelKey) == podLabel).ToArray();
+        }
 
-            var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray();
+        private PodInfo CreatePodInfo(V1Pod pod)
-            if (newPods.Length != 1) throw new InvalidOperationException("Expected only 1 pod to be created. Test infra failure.");
+        {
-
+            var name = pod.Name();
-            var newPod = newPods.Single();
+            var ip = pod.Status.PodIP;
-            var name = newPod.Name();
+            var k8sNodeName = pod.Spec.NodeName;
-            var ip = newPod.Status.PodIP;
-            var k8sNodeName = newPod.Spec.NodeName;
 
             if (string.IsNullOrEmpty(name)) throw new InvalidOperationException("Invalid pod name received. Test infra failure.");
             if (string.IsNullOrEmpty(ip)) throw new InvalidOperationException("Invalid pod IP received. Test infra failure.");
 
-            knownPods.Add(name);
             return new PodInfo(name, ip, k8sNodeName);
         }
     }

Framework/KubernetesWorkflow/K8sHooks.cs

@@ -1,4 +1,7 @@
-namespace KubernetesWorkflow
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
+
+namespace KubernetesWorkflow
 {
     public interface IK8sHooks
     {

Framework/KubernetesWorkflow/K8sNameUtils.cs

@@ -3,8 +3,19 @@
     public static class K8sNameUtils
     {
         public static string Format(string s)
+        {
+            return Format(s, 62);
+        }
+
+        public static string FormatPortName(string s)
+        {
+            return Format(s, 15);
+        }
+
+        private static string Format(string s, int maxLength)
         {
             var result = s.ToLowerInvariant()
+                .Replace("_", "-")
                 .Replace(" ", "-")
                 .Replace(":", "-")
                 .Replace("/", "-")
@@ -14,7 +25,7 @@
                 .Replace(",", "-");
 
             result = result.Trim('-');
-            if (result.Length > 62) result = result.Substring(0, 62);
+            if (result.Length > maxLength) result = result.Substring(0, maxLength);
 
             return result;
         }

Framework/KubernetesWorkflow/KnownK8sPods.cs

@@ -1,17 +0,0 @@
-namespace KubernetesWorkflow
-{
-    public class KnownK8sPods
-    {
-        private readonly List<string> knownActivePodNames = new List<string>();
-
-        public bool Contains(string name)
-        {
-            return knownActivePodNames.Contains(name);
-        }
-
-        public void Add(string name)
-        {
-            knownActivePodNames.Add(name);
-        }
-    }
-}

Framework/KubernetesWorkflow/Location.cs

@@ -1,4 +1,6 @@
-namespace KubernetesWorkflow
+using KubernetesWorkflow.Types;
+
+namespace KubernetesWorkflow
 {
     public interface ILocation
     {

Framework/KubernetesWorkflow/LocationProvider.cs

@@ -1,4 +1,5 @@
-using Logging;
+using KubernetesWorkflow.Types;
+using Logging;
 
 namespace KubernetesWorkflow
 {

Framework/KubernetesWorkflow/LogHandler.cs

@@ -0,0 +1,23 @@
+namespace KubernetesWorkflow
+{
+    public interface ILogHandler
+    {
+        void Log(Stream log);
+    }
+
+    public abstract class LogHandler : ILogHandler
+    {
+        public void Log(Stream log)
+        {
+            using var reader = new StreamReader(log);
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                ProcessLine(line);
+                line = reader.ReadLine();
+            }
+        }
+
+        protected abstract void ProcessLine(string line);
+    }
+}

Framework/KubernetesWorkflow/PublicIpService.cs

@@ -0,0 +1,7 @@
+namespace KubernetesWorkflow
+{
+    public static class PublicIpService
+    {
+        public static string Address { get; } = "ip.codex.storage";
+    }
+}

Framework/KubernetesWorkflow/Recipe/ContainerAdditionals.cs

@@ -1,6 +1,6 @@
 using Newtonsoft.Json.Linq;
 
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class ContainerAdditionals
     {
@@ -20,7 +20,7 @@ namespace KubernetesWorkflow
         {
             var typeName = GetTypeName(typeof(T));
             var userData = Additionals.SingleOrDefault(a => a.Type == typeName);
-            if (userData == null) return default(T);
+            if (userData == null) return default;
             var jobject = (JObject)userData.UserData;
             return jobject.ToObject<T>();
         }

Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs

@@ -1,13 +1,15 @@
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class ContainerRecipe
     {
-        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
+        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
         {
             Number = number;
             NameOverride = nameOverride;
             Image = image;
             Resources = resources;
+            SchedulingAffinity = schedulingAffinity;
+            SetCriticalPriority = setCriticalPriority;
             ExposedPorts = exposedPorts;
             InternalPorts = internalPorts;
             EnvVars = envVars;
@@ -24,12 +26,16 @@
             {
                 Name = $"ctnr{Number}";
             }
+
+            if (exposedPorts.Any(p => string.IsNullOrEmpty(p.Tag))) throw new Exception("Port tags are required for all exposed ports.");
         }
 
         public string Name { get; }
         public int Number { get; }
         public string? NameOverride { get; }
         public ContainerResources Resources { get; }
+        public SchedulingAffinity SchedulingAffinity { get; }
+        public bool SetCriticalPriority { get; }
         public string Image { get; }
         public Port[] ExposedPorts { get; }
         public Port[] InternalPorts { get; }
@@ -51,20 +57,50 @@
                 $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
                 $"envVars: {string.Join(",", EnvVars.Select(v => v.ToString()))}, " +
                 $"limits: {Resources}, " +
+                $"affinity: {SchedulingAffinity}, " +
                 $"volumes: {string.Join(",", Volumes.Select(v => $"'{v.MountPath}'"))}";
         }
     }
 
     public class Port
     {
-        public Port(int number, string tag)
+        public Port(int number, string tag, PortProtocol protocol)
         {
             Number = number;
             Tag = tag;
+            Protocol = protocol;
+
+            if (string.IsNullOrWhiteSpace(Tag))
+            {
+                throw new Exception("A unique port tag is required");
+            }
         }
 
         public int Number { get; }
         public string Tag { get; }
+        public PortProtocol Protocol { get; }
+
+        public bool IsTcp()
+        {
+            return Protocol == PortProtocol.TCP;
+        }
+
+        public bool IsUdp()
+        {
+            return Protocol == PortProtocol.UDP;
+        }
+
+        public override string ToString()
+        {
+            if (string.IsNullOrEmpty(Tag)) return $"untagged-port={Number}/{Protocol}";
+            return $"{Tag}={Number}/{Protocol}";
+        }
+    }
+
+    public enum PortProtocol
+    {
+        TCP,
+        UDP
     }
 
     public class EnvVar
@@ -86,15 +122,21 @@
 
     public class VolumeMount
     {
-        public VolumeMount(string volumeName, string mountPath, string resourceQuantity)
+        public VolumeMount(string volumeName, string mountPath, string? subPath = null, string? resourceQuantity = null, string? secret = null, string? hostPath = null)
         {
             VolumeName = volumeName;
             MountPath = mountPath;
+            SubPath = subPath;
             ResourceQuantity = resourceQuantity;
+            Secret = secret;
+            HostPath = hostPath;
         }
 
         public string VolumeName { get; }
         public string MountPath { get; }
-        public string ResourceQuantity { get; }
+        public string? SubPath { get; }
+        public string? ResourceQuantity { get; }
+        public string? Secret { get; }
+        public string? HostPath { get; }
     }
 }

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -1,6 +1,6 @@
 using Utils;
 
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public abstract class ContainerRecipeFactory
     {
@@ -13,6 +13,8 @@ namespace KubernetesWorkflow
         private readonly List<object> additionals = new List<object>();
         private RecipeComponentFactory factory = null!;
         private ContainerResources resources = new ContainerResources();
+        private SchedulingAffinity schedulingAffinity = new SchedulingAffinity();
+        private bool setCriticalPriority;
 
         public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config)
         {
@@ -22,7 +24,7 @@ namespace KubernetesWorkflow
 
             Initialize(config);
 
-            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources,
+            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, setCriticalPriority,
                 exposedPorts.ToArray(),
                 internalPorts.ToArray(),
                 envVars.ToArray(),
@@ -40,6 +42,8 @@ namespace KubernetesWorkflow
             additionals.Clear();
             this.factory = null!;
             resources = new ContainerResources();
+            schedulingAffinity = new SchedulingAffinity();
+            setCriticalPriority = false;
 
             return recipe;
         }
@@ -50,31 +54,31 @@ namespace KubernetesWorkflow
         protected int Index { get; private set; } = 0;
         protected abstract void Initialize(StartupConfig config);
 
-        protected Port AddExposedPort(string tag = "")
+        protected Port AddExposedPort(string tag, PortProtocol protocol = PortProtocol.TCP)
         {
-            return AddExposedPort(factory.CreatePort(tag));
+            return AddExposedPort(factory.CreateExternalPort(tag, protocol));
         }
 
-        protected Port AddExposedPort(int number, string tag = "")
+        protected Port AddExposedPort(int number, string tag, PortProtocol protocol = PortProtocol.TCP)
         {
-            return AddExposedPort(factory.CreatePort(number, tag));
+            return AddExposedPort(factory.CreateExternalPort(number, tag, protocol));
         }
 
-        protected Port AddInternalPort(string tag = "")
+        protected Port AddInternalPort(string tag = "", PortProtocol protocol = PortProtocol.TCP)
         {
-            var p = factory.CreatePort(tag);
+            var p = factory.CreateInternalPort(tag, protocol);
             internalPorts.Add(p);
             return p;
         }
 
-        protected void AddExposedPortAndVar(string name, string tag = "")
+        protected void AddExposedPortAndVar(string name, string tag, PortProtocol protocol = PortProtocol.TCP)
         {
-            AddEnvVar(name, AddExposedPort(tag));
+            AddEnvVar(name, AddExposedPort(tag, protocol));
         }
 
-        protected void AddInternalPortAndVar(string name, string tag = "")
+        protected void AddInternalPortAndVar(string name, string tag = "", PortProtocol protocol = PortProtocol.TCP)
         {
-            AddEnvVar(name, AddInternalPort(tag));
+            AddEnvVar(name, AddInternalPort(tag, protocol));
         }
 
         protected void AddEnvVar(string name, string value)
@@ -97,12 +101,18 @@ namespace KubernetesWorkflow
             podAnnotations.Add(name, value);
         }
 
+        protected void AddVolume(string name, string mountPath, string? subPath = null, string? secret = null, string? hostPath = null)
+        {
+            var size = 10.MB().ToSuffixNotation();
+            volumeMounts.Add(new VolumeMount(name, mountPath, subPath, size, secret, hostPath));
+        }
+
         protected void AddVolume(string mountPath, ByteSize volumeSize)
         {
             volumeMounts.Add(new VolumeMount(
                 $"autovolume-{Guid.NewGuid().ToString().ToLowerInvariant()}",
                 mountPath,
-                volumeSize.ToSuffixNotation()));
+                resourceQuantity: volumeSize.ToSuffixNotation()));
         }
 
         protected void Additional(object userData)
@@ -115,11 +125,23 @@ namespace KubernetesWorkflow
             SetResourcesRequest(new ContainerResourceSet(milliCPUs, memory));
         }
 
-        protected void SetResourceLimits(int milliCPUs, ByteSize memory)
+        protected void SetSchedulingAffinity(string notIn)
         {
-            SetResourceLimits(new ContainerResourceSet(milliCPUs, memory));
+            schedulingAffinity = new SchedulingAffinity(notIn);
         }
 
+        protected void SetSystemCriticalPriority()
+        {
+            setCriticalPriority = true;
+        }
+
+        // Disabled following a possible bug in the k8s cluster that will throttle containers much more than is
+        // called for if they have resource limits defined.
+        //protected void SetResourceLimits(int milliCPUs, ByteSize memory)
+        //{
+        //    SetResourceLimits(new ContainerResourceSet(milliCPUs, memory));
+        //}
+
         protected void SetResourcesRequest(ContainerResourceSet requests)
         {
             resources.Requests = requests;
@@ -132,11 +154,6 @@ namespace KubernetesWorkflow
 
         private Port AddExposedPort(Port port)
         {
-            if (exposedPorts.Any())
-            {
-                throw new NotImplementedException("Current implementation only support 1 exposed port per container recipe. " +
-                    $"Methods for determining container addresses in {nameof(StartupWorkflow)} currently rely on this constraint.");
-            }
             exposedPorts.Add(port);
             return port;
         }

Framework/KubernetesWorkflow/Recipe/ContainerResources.cs

@@ -1,6 +1,6 @@
 using Utils;
 
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class ContainerResources
     {

Framework/KubernetesWorkflow/Recipe/PodAnnotations.cs

@@ -1,4 +1,4 @@
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class PodAnnotations
     {

Framework/KubernetesWorkflow/Recipe/PodLabels.cs

@@ -1,4 +1,4 @@
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class PodLabels
     {

Framework/KubernetesWorkflow/Recipe/RecipeComponentFactory.cs

@@ -0,0 +1,50 @@
+using System.Globalization;
+using Utils;
+
+namespace KubernetesWorkflow.Recipe
+{
+    public class RecipeComponentFactory
+    {
+        private readonly NumberSource internalNumberSource = new NumberSource(8080);
+        private static readonly NumberSource externalNumberSource = new NumberSource(30000);
+        private static int[] usedExternalPorts = Array.Empty<int>();
+
+        public void Update(K8sController controller)
+        {
+            usedExternalPorts = controller.GetUsedExternalPorts();
+        }
+
+        public Port CreateInternalPort(string tag, PortProtocol protocol)
+        {
+            return new Port(internalNumberSource.GetNextNumber(), tag, protocol);
+        }
+
+        public Port CreateExternalPort(int number, string tag, PortProtocol protocol)
+        {
+            if (usedExternalPorts.Contains(number)) throw new Exception($"External port number {number} is already in use by the cluster.");
+            return new Port(number, tag, protocol);
+        }
+
+        public Port CreateExternalPort(string tag, PortProtocol protocol)
+        {
+            while (true)
+            {
+                var number = externalNumberSource.GetNextNumber();
+                if (!usedExternalPorts.Contains(number))
+                {
+                    return new Port(number, tag, protocol);
+                }
+            }
+        }
+
+        public EnvVar CreateEnvVar(string name, int value)
+        {
+            return CreateEnvVar(name, value.ToString(CultureInfo.InvariantCulture));
+        }
+
+        public EnvVar CreateEnvVar(string name, string value)
+        {
+            return new EnvVar(name, value);
+        }
+    }
+}

Framework/KubernetesWorkflow/Recipe/SchedulingAffinity.cs

@@ -0,0 +1,18 @@
+namespace KubernetesWorkflow.Recipe
+{
+    public class SchedulingAffinity
+    {
+        public SchedulingAffinity(string? notIn = null)
+        {
+            NotIn = notIn;
+        }
+
+        public string? NotIn { get; }
+
+        public override string ToString()
+        {
+            if (string.IsNullOrEmpty(NotIn)) return "none";
+            return "notIn:" + NotIn;
+        }
+    }
+}

Framework/KubernetesWorkflow/RecipeComponentFactory.cs

@@ -1,30 +0,0 @@
-using System.Globalization;
-using Utils;
-
-namespace KubernetesWorkflow
-{
-    public class RecipeComponentFactory
-    {
-        private NumberSource portNumberSource = new NumberSource(8080);
-
-        public Port CreatePort(int number, string tag)
-        {
-            return new Port(number, tag);
-        }
-
-        public Port CreatePort(string tag)
-        {
-            return new Port(portNumberSource.GetNextNumber(), tag);
-        }
-
-        public EnvVar CreateEnvVar(string name, int value)
-        {
-            return CreateEnvVar(name, value.ToString(CultureInfo.InvariantCulture));
-        }
-
-        public EnvVar CreateEnvVar(string name, string value)
-        {
-            return new EnvVar(name, value);
-        }
-    }
-}

Framework/KubernetesWorkflow/RunnerLocationUtils.cs

@@ -1,55 +1,38 @@
-using System.Net.NetworkInformation;
+namespace KubernetesWorkflow
-using Utils;
-
-namespace KubernetesWorkflow
 {
     internal enum RunnerLocation
     {
+        Unknown,
         ExternalToCluster,
         InternalToCluster,
     }
 
     internal static class RunnerLocationUtils
     {
-        private static RunnerLocation? knownLocation = null;
+        private static RunnerLocation location = RunnerLocation.Unknown;
 
-        internal static RunnerLocation DetermineRunnerLocation(RunningContainer container)
+        internal static RunnerLocation GetRunnerLocation()
         {
-            if (knownLocation != null) return knownLocation.Value;
+            DetermineRunnerLocation();
-
+            if (location == RunnerLocation.Unknown) throw new Exception("Runner location is unknown.");
-            if (PingHost(container.Pod.PodInfo.Ip))
+            return location;
-            {
-                knownLocation = RunnerLocation.InternalToCluster;
-            }
-            else if (PingHost(Format(container.ClusterExternalAddress)))
-            {
-                knownLocation = RunnerLocation.ExternalToCluster;
-            }
-
-            if (knownLocation == null) throw new Exception("Unable to determine location relative to kubernetes cluster.");
-            return knownLocation.Value;
         }
 
-        private static string Format(Address host)
+        private static void DetermineRunnerLocation()
         {
-            return host.Host
+            if (location != RunnerLocation.Unknown) return;
-                .Replace("http://", "")
-                .Replace("https://", "");
-        }
 
-        private static bool PingHost(string host)
+            var port = Environment.GetEnvironmentVariable("KUBERNETES_PORT");
-        {
+            var host = Environment.GetEnvironmentVariable("KUBERNETES_SERVICE_HOST");
-            try
-            {
-                using var pinger = new Ping();
-                PingReply reply = pinger.Send(host);
-                return reply.Status == IPStatus.Success;
-            }
-            catch (PingException)
-            {
-            }
 
-            return false;
+            if (string.IsNullOrEmpty(port) || string.IsNullOrEmpty(host))
+            {
+                location = RunnerLocation.ExternalToCluster;
+            }
+            else
+            {
+                location = RunnerLocation.InternalToCluster;
+            }
         }
     }
 }

Framework/KubernetesWorkflow/RunningContainers.cs

@@ -1,70 +0,0 @@
-using Newtonsoft.Json;
-using Utils;
-
-namespace KubernetesWorkflow
-{
-    public class RunningContainers
-    {
-        public RunningContainers(StartupConfig startupConfig, RunningPod runningPod, RunningContainer[] containers)
-        {
-            StartupConfig = startupConfig;
-            RunningPod = runningPod;
-            Containers = containers;
-        }
-
-        public StartupConfig StartupConfig { get; }
-        public RunningPod RunningPod { get; }
-        public RunningContainer[] Containers { get; }
-
-        public string Describe()
-        {
-            return string.Join(",", Containers.Select(c => c.Name));
-        }
-    }
-
-    public class RunningContainer
-    {
-        public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, Address clusterExternalAddress, Address clusterInternalAddress)
-        {
-            Pod = pod;
-            Recipe = recipe;
-            ServicePorts = servicePorts;
-            Name = name;
-            ClusterExternalAddress = clusterExternalAddress;
-            ClusterInternalAddress = clusterInternalAddress;
-        }
-
-        public string Name { get; }
-        public RunningPod Pod { get; }
-        public ContainerRecipe Recipe { get; }
-        public Port[] ServicePorts { get; }
-        public Address ClusterExternalAddress { get; }
-        public Address ClusterInternalAddress { get; }
-
-        [JsonIgnore]
-        public Address Address
-        {
-            get
-            {
-                if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster)
-                {
-                    return ClusterInternalAddress;
-                }
-                return ClusterExternalAddress;
-            }
-        }
-    }
-
-    public static class RunningContainersExtensions
-    {
-        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
-        {
-            return runningContainers.SelectMany(c => c.Containers).ToArray();
-        }
-
-        public static string Describe(this RunningContainers[] runningContainers)
-        {
-            return string.Join(",", runningContainers.Select(c => c.Describe()));
-        }
-    }
-}

Framework/KubernetesWorkflow/RunningPod.cs

@@ -1,56 +1,69 @@
-namespace KubernetesWorkflow
+using k8s;
+using k8s.Models;
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
+using Newtonsoft.Json;
+
+namespace KubernetesWorkflow
 {
-    public class RunningPod
+    public class StartResult
     {
-        public RunningPod(K8sCluster cluster, PodInfo podInfo, string deploymentName, string serviceName,  ContainerRecipePortMapEntry[] portMapEntries)
+        public StartResult(
+            K8sCluster cluster,
+            ContainerRecipe[] containerRecipes, 
+            RunningDeployment deployment,
+            RunningService? internalService,
+            RunningService? externalService)
         {
             Cluster = cluster;
-            PodInfo = podInfo;
+            ContainerRecipes = containerRecipes;
-            DeploymentName = deploymentName;
+            Deployment = deployment;
-            ServiceName = serviceName;
+            InternalService = internalService;
-            PortMapEntries = portMapEntries;
+            ExternalService = externalService;
         }
 
         public K8sCluster Cluster { get; }
-        public PodInfo PodInfo { get; }
+        public ContainerRecipe[] ContainerRecipes { get; }
-        public ContainerRecipePortMapEntry[] PortMapEntries { get; }
+        public RunningDeployment Deployment { get; }
-        public string DeploymentName { get; }
+        public RunningService? InternalService { get; }
-        public string ServiceName { get; }
+        public RunningService? ExternalService { get; }
 
-        public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
+        public Port GetInternalServicePorts(ContainerRecipe recipe, string tag)
         {
-            if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number))
+            if (InternalService != null)
             {
-                return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports;
+                var p = InternalService.GetServicePortForRecipeAndTag(recipe, tag);
+                if (p != null) return p;
+            }
+
+            throw new Exception($"Unable to find internal port by tag '{tag}' for recipe '{recipe.Name}'.");
+        }
+
+        public Port GetExternalServicePorts(ContainerRecipe recipe, string tag)
+        {
+            if (ExternalService != null)
+            {
+                var p = ExternalService.GetServicePortForRecipeAndTag(recipe, tag);
+                if (p != null) return p;
+            }
+
+            throw new Exception($"Unable to find external port by tag '{tag}' for recipe '{recipe.Name}'.");
+        }
+
+        public Port[] GetServicePortsForContainer(ContainerRecipe recipe)
+        {
+            if (InternalService != null)
+            {
+                var p = InternalService.GetServicePortsForRecipe(recipe);
+                if (p.Any()) return p;
+            }
+            if (ExternalService != null)
+            {
+                var p = ExternalService.GetServicePortsForRecipe(recipe);
+                if (p.Any()) return p;
             }
 
             return Array.Empty<Port>();
         }
     }
-
-    public class ContainerRecipePortMapEntry
-    {
-        public ContainerRecipePortMapEntry(int containerNumber, Port[] ports)
-        {
-            ContainerNumber = containerNumber;
-            Ports = ports;
-        }
-
-        public int ContainerNumber { get; }
-        public Port[] Ports { get; }
-    }
-
-    public class PodInfo
-    {
-        public PodInfo(string name, string ip, string k8sNodeName)
-        {
-            Name = name;
-            Ip = ip;
-            K8SNodeName = k8sNodeName;
-        }
-
-        public string Name { get; }
-        public string Ip { get; }
-        public string K8SNodeName { get; }
-    }
 }

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -1,4 +1,7 @@
-using Logging;
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
+using Logging;
+using Newtonsoft.Json;
 using Utils;
 
 namespace KubernetesWorkflow
@@ -8,8 +11,10 @@ namespace KubernetesWorkflow
         IKnownLocations GetAvailableLocations();
         RunningContainers Start(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
         RunningContainers Start(int numberOfContainers, ILocation location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig);
+        PodInfo GetPodInfo(RunningContainer container);
+        PodInfo GetPodInfo(RunningContainers containers);
         CrashWatcher CreateCrashWatcher(RunningContainer container);
-        void Stop(RunningContainers runningContainers);
+        void Stop(RunningContainers containers);
         void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines = null);
         string ExecuteCommand(RunningContainer container, string command, params string[] args);
         void DeleteNamespace();
@@ -21,17 +26,15 @@ namespace KubernetesWorkflow
         private readonly ILog log;
         private readonly WorkflowNumberSource numberSource;
         private readonly K8sCluster cluster;
-        private readonly KnownK8sPods knownK8SPods;
         private readonly string k8sNamespace;
         private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
         private readonly LocationProvider locationProvider;
 
-        internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string k8sNamespace)
+        internal StartupWorkflow(ILog log, WorkflowNumberSource numberSource, K8sCluster cluster, string k8sNamespace)
         {
             this.log = log;
             this.numberSource = numberSource;
             this.cluster = cluster;
-            this.knownK8SPods = knownK8SPods;
             this.k8sNamespace = k8sNamespace;
 
             locationProvider = new LocationProvider(log, K8s);
@@ -51,16 +54,33 @@ namespace KubernetesWorkflow
         {
             return K8s(controller =>
             {
-                var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
+                componentFactory.Update(controller);
-                var runningPod = controller.BringOnline(recipes, location);
-                var containers = CreateContainers(runningPod, recipes, startupConfig);
 
-                var rc = new RunningContainers(startupConfig, runningPod, containers);
+                var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
+                var startResult = controller.BringOnline(recipes, location);
+                var containers = CreateContainers(startResult, recipes, startupConfig);
+
+                var rc = new RunningContainers(startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
+
+                if (startResult.ExternalService != null)
+                {
+                    componentFactory.Update(controller);
+                }
                 return rc;
             });
         }
 
+        public PodInfo GetPodInfo(RunningContainer container)
+        {
+            return K8s(c => c.GetPodInfo(container.RunningContainers.StartResult.Deployment));
+        }
+
+        public PodInfo GetPodInfo(RunningContainers containers)
+        {
+            return K8s(c => c.GetPodInfo(containers.StartResult.Deployment));
+        }
+
         public CrashWatcher CreateCrashWatcher(RunningContainer container)
         {
             return K8s(c => c.CreateCrashWatcher(container));
@@ -70,7 +90,7 @@ namespace KubernetesWorkflow
         {
             K8s(controller =>
             {
-                controller.Stop(runningContainers.RunningPod);
+                controller.Stop(runningContainers.StartResult);
                 cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
             });
         }
@@ -79,7 +99,7 @@ namespace KubernetesWorkflow
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler, tailLines);
+                controller.DownloadPodLog(container, logHandler, tailLines);
             });
         }
 
@@ -87,7 +107,7 @@ namespace KubernetesWorkflow
         {
             return K8s(controller =>
             {
-                return controller.ExecuteCommand(container.Pod, container.Recipe.Name, command, args);
+                return controller.ExecuteCommand(container, command, args);
             });
         }
 
@@ -107,19 +127,16 @@ namespace KubernetesWorkflow
             });
         }
 
-        private RunningContainer[] CreateContainers(RunningPod runningPod, ContainerRecipe[] recipes, StartupConfig startupConfig)
+        private RunningContainer[] CreateContainers(StartResult startResult, ContainerRecipe[] recipes, StartupConfig startupConfig)
         {
             log.Debug();
             return recipes.Select(r =>
             {
-                var servicePorts = runningPod.GetServicePortsForContainerRecipe(r);
-                log.Debug($"{r} -> service ports: {string.Join(",", servicePorts.Select(p => p.Number))}");
-
                 var name = GetContainerName(r, startupConfig);
+                var addresses = CreateContainerAddresses(startResult, r);
+                log.Debug($"{r}={name} -> container addresses: {string.Join(Environment.NewLine, addresses.Select(a => a.ToString()))}");
 
-                return new RunningContainer(runningPod, r, servicePorts, name,
+                return new RunningContainer(name, r, addresses);
-                    GetContainerExternalAddress(runningPod, servicePorts),
-                    GetContainerInternalAddress(r));
 
             }).ToArray();
         }
@@ -137,35 +154,42 @@ namespace KubernetesWorkflow
             }
         }
 
-        private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
+        private ContainerAddress[] CreateContainerAddresses(StartResult startResult, ContainerRecipe recipe)
         {
-            return new Address(
+            var result = new List<ContainerAddress>();
-                pod.Cluster.HostAddress,
+            foreach (var exposedPort in recipe.ExposedPorts)
-                GetServicePort(servicePorts));
+            {
+                result.Add(new ContainerAddress(exposedPort.Tag, GetContainerExternalAddress(startResult, recipe, exposedPort.Tag), false));
+                result.Add(new ContainerAddress(exposedPort.Tag, GetContainerInternalAddress(startResult, recipe, exposedPort.Tag), true));
+            }
+            foreach (var internalPort in recipe.InternalPorts)
+            {
+                result.Add(new ContainerAddress(internalPort.Tag, GetContainerInternalAddress(startResult, recipe, internalPort.Tag), true));
+            }
+
+            return result.ToArray();
         }
 
-        private Address GetContainerInternalAddress(ContainerRecipe recipe)
+        private static Address GetContainerExternalAddress(StartResult startResult, ContainerRecipe recipe, string tag)
         {
-            var serviceName = "service-" + numberSource.WorkflowNumber;
+            var port = startResult.GetExternalServicePorts(recipe, tag);
-            var port = GetInternalPort(recipe);
 
             return new Address(
-                $"http://{serviceName}.{k8sNamespace}.svc.cluster.local",
+                startResult.Cluster.HostAddress,
-                port);
+                port.Number);
         }
 
-        private static int GetServicePort(Port[] servicePorts)
+        private Address GetContainerInternalAddress(StartResult startResult, ContainerRecipe recipe, string tag)
         {
-            if (servicePorts.Any()) return servicePorts.First().Number;
+            var namespaceName = startResult.Cluster.Configuration.KubernetesNamespace;
-            return 0;
+            var serviceName = startResult.InternalService!.Name;
-        }
+            var port = startResult.GetInternalServicePorts(recipe, tag);
 
-        private static int GetInternalPort(ContainerRecipe recipe)
+            return new Address(
-        {
+                $"http://{serviceName}.{namespaceName}.svc.cluster.local",
-            if (recipe.ExposedPorts.Any()) return recipe.ExposedPorts.First().Number;
+                port.Number);
-            return 0;
         }
-
+        
         private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
             log.Debug();
@@ -173,6 +197,8 @@ namespace KubernetesWorkflow
             for (var i = 0; i < numberOfContainers; i++)
             {
                 var recipe = recipeFactory.CreateRecipe(i, numberSource.GetContainerNumber(), componentFactory, startupConfig);
+                CheckPorts(recipe);
+
                 if (cluster.Configuration.AddAppPodLabel) recipe.PodLabels.Add("app", recipeFactory.AppName);
                 cluster.Configuration.Hooks.OnContainerRecipeCreated(recipe);
                 result.Add(recipe);
@@ -181,40 +207,47 @@ namespace KubernetesWorkflow
             return result.ToArray();
         }
 
+        private void CheckPorts(ContainerRecipe recipe)
+        {
+            var allTags =
+                recipe.ExposedPorts.Concat(recipe.InternalPorts)
+                .Select(p => K8sNameUtils.Format(p.Tag)).ToArray();
+
+            if (allTags.Length != allTags.Distinct().Count())
+            {
+                throw new Exception("Duplicate port tags found in recipe for " + recipe.Name);
+            }
+        }
+
         private void K8s(Action<K8sController> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, k8sNamespace);
+            try
-            action(controller);
+            {
-            controller.Dispose();
+                var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
+                action(controller);
+                controller.Dispose();
+            }
+            catch (k8s.Autorest.HttpOperationException ex)
+            {
+                log.Error(JsonConvert.SerializeObject(ex.Response));
+                throw;
+            }
         }
 
         private T K8s<T>(Func<K8sController, T> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, k8sNamespace);
+            try
-            var result = action(controller);
-            controller.Dispose();
-            return result;
-        }
-    }
-
-    public interface ILogHandler
-    {
-        void Log(Stream log);
-    }
-
-    public abstract class LogHandler : ILogHandler
-    {
-        public void Log(Stream log)
-        {
-            using var reader = new StreamReader(log);
-            var line = reader.ReadLine();
-            while (line != null)
             {
-                ProcessLine(line);
+                var controller = new K8sController(log, cluster, numberSource, k8sNamespace);
-                line = reader.ReadLine();
+                var result = action(controller);
+                controller.Dispose();
+                return result;
+            }
+            catch (k8s.Autorest.HttpOperationException ex)
+            {
+                log.Error(JsonConvert.SerializeObject(ex.Response));
+                throw;
             }
         }
-
-        protected abstract void ProcessLine(string line);
     }
 }

Framework/KubernetesWorkflow/Types/ContainerAddress.cs

@@ -0,0 +1,24 @@
+using Utils;
+
+namespace KubernetesWorkflow.Types
+{
+    public class ContainerAddress
+    {
+        public ContainerAddress(string portTag, Address address, bool isInteral)
+        {
+            PortTag = portTag;
+            Address = address;
+            IsInteral = isInteral;
+        }
+
+        public string PortTag { get; }
+        public Address Address { get; }
+        public bool IsInteral { get; }
+
+        public override string ToString()
+        {
+            var indicator = IsInteral ? "int" : "ext";
+            return $"{indicator} {PortTag} -> '{Address}'";
+        }
+    }
+}

Framework/KubernetesWorkflow/Types/ContainerRecipePortMapEntry.cs

@@ -0,0 +1,16 @@
+using KubernetesWorkflow.Recipe;
+
+namespace KubernetesWorkflow.Types
+{
+    public class ContainerRecipePortMapEntry
+    {
+        public ContainerRecipePortMapEntry(int recipeNumber, Port[] ports)
+        {
+            RecipeNumber = recipeNumber;
+            Ports = ports;
+        }
+
+        public int RecipeNumber { get; }
+        public Port[] Ports { get; }
+    }
+}

Framework/KubernetesWorkflow/Types/K8sNodeLabel.cs

@@ -0,0 +1,14 @@
+namespace KubernetesWorkflow.Types
+{
+    public class K8sNodeLabel
+    {
+        public K8sNodeLabel(string key, string value)
+        {
+            Key = key;
+            Value = value;
+        }
+
+        public string Key { get; }
+        public string Value { get; }
+    }
+}

Framework/KubernetesWorkflow/Types/PodInfo.cs

@@ -0,0 +1,16 @@
+namespace KubernetesWorkflow.Types
+{
+    public class PodInfo
+    {
+        public PodInfo(string name, string ip, string k8sNodeName)
+        {
+            Name = name;
+            Ip = ip;
+            K8SNodeName = k8sNodeName;
+        }
+
+        public string Name { get; }
+        public string Ip { get; }
+        public string K8SNodeName { get; }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -0,0 +1,54 @@
+using KubernetesWorkflow.Recipe;
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+
+namespace KubernetesWorkflow.Types
+{
+    public class RunningContainer
+    {
+        public RunningContainer(string name, ContainerRecipe recipe, ContainerAddress[] addresses)
+        {
+            Name = name;
+            Recipe = recipe;
+            Addresses = addresses;
+        }
+
+        public string Name { get; }
+        public ContainerRecipe Recipe { get; }
+        public ContainerAddress[] Addresses { get; }
+
+        [JsonIgnore]
+        public RunningContainers RunningContainers { get; internal set; } = null!;
+
+        public Address GetAddress(ILog log, string portTag)
+        {
+            var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
+            if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);
+
+            var select = SelectAddress(addresses);
+            log.Debug($"Container '{Name}' selected for tag '{portTag}' address: '{select}'");
+            return select.Address;
+        }
+
+        public Address GetInternalAddress(string portTag)
+        {
+            var containerAddress = Addresses.Single(a => a.PortTag == portTag && a.IsInteral);
+            return containerAddress.Address;
+        }
+
+        private ContainerAddress SelectAddress(ContainerAddress[] addresses)
+        {
+            var location = RunnerLocationUtils.GetRunnerLocation();
+            if (location == RunnerLocation.InternalToCluster)
+            {
+                return addresses.Single(a => a.IsInteral);
+            }
+            if (location == RunnerLocation.ExternalToCluster)
+            {
+                return addresses.Single(a => !a.IsInteral);
+            }
+            throw new Exception("Running location not known.");
+        }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningContainers.cs

@@ -0,0 +1,44 @@
+using Newtonsoft.Json;
+
+namespace KubernetesWorkflow.Types
+{
+    public class RunningContainers
+    {
+        public RunningContainers(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        {
+            StartupConfig = startupConfig;
+            StartResult = startResult;
+            Containers = containers;
+
+            foreach (var c in containers) c.RunningContainers = this;
+        }
+
+        public StartupConfig StartupConfig { get; }
+        public StartResult StartResult { get; }
+        public RunningContainer[] Containers { get; }
+
+        [JsonIgnore]
+        public string Name
+        {
+            get { return $"{Containers.Length}x '{Containers.First().Name}'"; }
+        }
+
+        public string Describe()
+        {
+            return string.Join(",", Containers.Select(c => c.Name));
+        }
+    }
+
+    public static class RunningContainersExtensions
+    {
+        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
+        {
+            return runningContainers.SelectMany(c => c.Containers).ToArray();
+        }
+
+        public static string Describe(this RunningContainers[] runningContainers)
+        {
+            return string.Join(",", runningContainers.Select(c => c.Describe()));
+        }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningDeployment.cs

@@ -0,0 +1,14 @@
+namespace KubernetesWorkflow.Types
+{
+    public class RunningDeployment
+    {
+        public RunningDeployment(string name, string podLabel)
+        {
+            Name = name;
+            PodLabel = podLabel;
+        }
+
+        public string Name { get; }
+        public string PodLabel { get; }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningService.cs

@@ -0,0 +1,29 @@
+using KubernetesWorkflow.Recipe;
+
+namespace KubernetesWorkflow.Types
+{
+    public class RunningService
+    {
+        public RunningService(string name, List<ContainerRecipePortMapEntry> result)
+        {
+            Name = name;
+            Result = result;
+        }
+
+        public string Name { get; }
+        public List<ContainerRecipePortMapEntry> Result { get; }
+
+        public Port? GetServicePortForRecipeAndTag(ContainerRecipe recipe, string tag)
+        {
+            return GetServicePortsForRecipe(recipe).SingleOrDefault(p => p.Tag == tag);
+        }
+
+        public Port[] GetServicePortsForRecipe(ContainerRecipe recipe)
+        {
+            return Result
+                .Where(p => p.RecipeNumber == recipe.Number)
+                .SelectMany(p => p.Ports)
+                .ToArray();
+        }
+    }
+}

Framework/KubernetesWorkflow/WorkflowCreator.cs

@@ -7,7 +7,6 @@ namespace KubernetesWorkflow
     {
         private readonly NumberSource numberSource = new NumberSource(0);
         private readonly NumberSource containerNumberSource = new NumberSource(0);
-        private readonly KnownK8sPods knownPods = new KnownK8sPods();
         private readonly K8sCluster cluster;
         private readonly ILog log;
         private readonly Configuration configuration;
@@ -26,7 +25,7 @@ namespace KubernetesWorkflow
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
                                                                     containerNumberSource);
 
-            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, GetNamespace(namespaceOverride));
+            return new StartupWorkflow(log, workflowNumberSource, cluster, GetNamespace(namespaceOverride));
         }
 
         private string GetNamespace(string? namespaceOverride)

Framework/Logging/BaseLog.cs

@@ -13,16 +13,21 @@ namespace Logging
 
     public abstract class BaseLog : ILog
     {
+        public static bool EnableDebugLogging { get; set; } = false;
+
         private readonly NumberSource subfileNumberSource = new NumberSource(0);
-        private readonly bool debug;
         private readonly List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
         private LogFile? logFile;
 
-        protected BaseLog(bool debug)
+        public BaseLog()
         {
-            this.debug = debug;
+            IsDebug =
+                EnableDebugLogging ||
+                !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("LOGDEBUG")) ||
+                !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("DEBUGLOG"));
         }
 
+        protected bool IsDebug { get; private set; }
         protected abstract string GetFullName();
 
         public LogFile LogFile 
@@ -39,19 +44,20 @@ namespace Logging
             LogFile.Write(ApplyReplacements(message));
         }
 
-        public virtual void Debug(string message = "", int skipFrames = 0)
+        public void Debug(string message = "", int skipFrames = 0)
         {
-            if (debug)
+            if (IsDebug)
             {
                 var callerName = DebugStack.GetCallerName(skipFrames);
-                // We don't use Log because in the debug output we should not have any replacements.
+                Log($"(debug)({callerName}) {message}");
-                LogFile.Write($"(debug)({callerName}) {message}");
             }
         }
 
         public virtual void Error(string message)
         {
-            Log($"[ERROR] {message}");
+            var msg = $"[ERROR] {message}";
+            Console.WriteLine(msg);
+            Log(msg);
         }
 
         public virtual void AddStringReplace(string from, string to)
@@ -73,6 +79,7 @@ namespace Logging
 
         private string ApplyReplacements(string str)
         {
+            if (IsDebug) return str;
             foreach (var replacement in replacements)
             {
                 str = replacement.Apply(str);

Framework/Logging/ConsoleLog.cs

@@ -2,10 +2,6 @@
 {
     public class ConsoleLog : BaseLog
     {
-        public ConsoleLog() : base(false)
-        {
-        }
-
         protected override string GetFullName()
         {
             return "CONSOLE";

Framework/Logging/FileLog.cs

@@ -0,0 +1,17 @@
+namespace Logging
+{
+    public class FileLog : BaseLog
+    {
+        public FileLog(string fullFilename)
+        {
+            FullFilename = fullFilename;
+        }
+
+        public string FullFilename { get; }
+
+        protected override string GetFullName()
+        {
+            return FullFilename;
+        }
+    }
+}

Framework/Logging/LogConfig.cs

@@ -2,13 +2,11 @@
 {
     public class LogConfig
     {
-        public LogConfig(string logRoot, bool debugEnabled)
+        public LogConfig(string logRoot)
         {
             LogRoot = logRoot;
-            DebugEnabled = debugEnabled;
         }
         
         public string LogRoot { get; }
-        public bool DebugEnabled { get; }
     }
 }

Framework/Logging/NullLog.cs

@@ -2,10 +2,6 @@
 {
     public class NullLog : BaseLog
     {
-        public NullLog() : base(false)
-        {
-        }
-
         public string FullFilename { get; set; } = "NULL";
 
         protected override string GetFullName()
@@ -15,15 +11,13 @@
 
         public override void Log(string message)
         {
-        }
+            if (IsDebug) base.Log(message);
-
-        public override void Debug(string message = "", int skipFrames = 0)
-        {
         }
 
         public override void Error(string message)
         {
             Console.WriteLine("Error: " + message);
+            base.Error(message);
         }
 
         public override void AddStringReplace(string from, string to)

Framework/Logging/Stopwatch.cs

@@ -16,19 +16,19 @@ namespace Logging
             this.debug = debug;
         }
 
-        public static void Measure(ILog log, string name, Action action, bool debug = false)
+        public static TimeSpan Measure(ILog log, string name, Action action, bool debug = false)
         {
             var sw = Begin(log, name, debug);
             action();
-            sw.End();
+            return sw.End();
         }
 
-        public static T Measure<T>(ILog log, string name, Func<T> action, bool debug = false)
+        public static StopwatchResult<T> Measure<T>(ILog log, string name, Func<T> action, bool debug = false)
         {
             var sw = Begin(log, name, debug);
             var result = action();
-            sw.End();
+            var duration = sw.End();
-            return result;
+            return new StopwatchResult<T>(result, duration);
         }
 
         public static Stopwatch Begin(ILog log)
@@ -68,4 +68,16 @@ namespace Logging
             return duration;
         }
     }
+
+    public class StopwatchResult<T>
+    {
+        public StopwatchResult(T value, TimeSpan duration)
+        {
+            Value = value;
+            Duration = duration;
+        }
+
+        public T Value { get; }
+        public TimeSpan Duration { get; }
+    }
 }

Framework/NethereumWorkflow/BlockTimeEntry.cs

@@ -0,0 +1,22 @@
+namespace NethereumWorkflow
+{
+    public partial class BlockTimeFinder
+    {
+        public class BlockTimeEntry
+        {
+            public BlockTimeEntry(ulong blockNumber, DateTime utc)
+            {
+                BlockNumber = blockNumber;
+                Utc = utc;
+            }
+
+            public ulong BlockNumber { get; }
+            public DateTime Utc { get; }
+
+            public override string ToString()
+            {
+                return $"[{BlockNumber}] @ {Utc.ToString("o")}";
+            }
+        }
+    }
+}

Framework/NethereumWorkflow/BlockTimeFinder.cs

@@ -0,0 +1,280 @@
+using Logging;
+using Nethereum.RPC.Eth.DTOs;
+using Nethereum.Web3;
+using Utils;
+
+namespace NethereumWorkflow
+{
+    public partial class BlockTimeFinder
+    {
+        private const ulong FetchRange = 6;
+        private const int MaxEntries = 1024;
+        private static readonly Dictionary<ulong, BlockTimeEntry> entries = new Dictionary<ulong, BlockTimeEntry>();
+        private readonly Web3 web3;
+        private readonly ILog log;
+        
+        public BlockTimeFinder(Web3 web3, ILog log)
+        {
+            this.web3 = web3;
+            this.log = log;
+        }
+
+        public ulong GetHighestBlockNumberBefore(DateTime moment)
+        {
+            log.Log("Looking for highest block before " + moment.ToString("o"));
+            AssertMomentIsInPast(moment);
+            Initialize();
+
+            return GetHighestBlockBefore(moment);
+        }
+
+        public ulong GetLowestBlockNumberAfter(DateTime moment)
+        {
+            log.Log("Looking for lowest block after " + moment.ToString("o"));
+            AssertMomentIsInPast(moment);
+            Initialize();
+
+            return GetLowestBlockAfter(moment);
+        }
+
+        private ulong GetHighestBlockBefore(DateTime moment)
+        {
+            var closestBefore = FindClosestBeforeEntry(moment);
+            var closestAfter = FindClosestAfterEntry(moment);
+
+            if (closestBefore != null &&
+                closestAfter != null &&
+                closestBefore.Utc < moment &&
+                closestAfter.Utc > moment &&
+                closestBefore.BlockNumber + 1 == closestAfter.BlockNumber)
+            {
+                log.Log("Found highest-Before: " + closestBefore);
+                return closestBefore.BlockNumber;
+            }
+
+            FetchBlocksAround(moment);
+            return GetHighestBlockBefore(moment);
+        }
+
+        private ulong GetLowestBlockAfter(DateTime moment)
+        {
+            var closestBefore = FindClosestBeforeEntry(moment);
+            var closestAfter = FindClosestAfterEntry(moment);
+
+            if (closestBefore != null &&
+                closestAfter != null &&
+                closestBefore.Utc < moment &&
+                closestAfter.Utc > moment &&
+                closestBefore.BlockNumber + 1 == closestAfter.BlockNumber)
+            {
+                log.Log("Found lowest-after: " + closestAfter);
+                return closestAfter.BlockNumber;
+            }
+
+            FetchBlocksAround(moment);
+            return GetLowestBlockAfter(moment);
+        }
+
+        private void FetchBlocksAround(DateTime moment)
+        {
+            var timePerBlock = EstimateTimePerBlock();
+            log.Debug("Fetching blocks around " + moment.ToString("o") + " timePerBlock: " + timePerBlock.TotalSeconds);
+
+            EnsureRecentBlockIfNecessary(moment, timePerBlock);
+
+            var max = entries.Keys.Max();
+            var blockDifference = CalculateBlockDifference(moment, timePerBlock, max);
+
+            FetchUp(max, blockDifference);
+            FetchDown(max, blockDifference);
+        }
+
+        private void FetchDown(ulong max, ulong blockDifference)
+        {
+            var target = max - blockDifference - 1;
+            var fetchDown = FetchRange;
+            while (fetchDown > 0)
+            {
+                if (!entries.ContainsKey(target))
+                {
+                    var newBlock = AddBlockNumber(target);
+                    if (newBlock == null) return;
+                    fetchDown--;
+                }
+                target--;
+                if (target <= 0) return;
+            }
+        }
+
+        private void FetchUp(ulong max, ulong blockDifference)
+        {
+            var target = max - blockDifference;
+            var fetchUp = FetchRange;
+            while (fetchUp > 0)
+            {
+                if (!entries.ContainsKey(target))
+                {
+                    var newBlock = AddBlockNumber(target);
+                    if (newBlock == null) return;
+                    fetchUp--;
+                }
+                target++;
+                if (target >= max) return;
+            }
+        }
+
+        private ulong CalculateBlockDifference(DateTime moment, TimeSpan timePerBlock, ulong max)
+        {
+            var latest = entries[max];
+            var timeDifference = latest.Utc - moment;
+            double secondsDifference = Math.Abs(timeDifference.TotalSeconds);
+            double secondsPerBlock = timePerBlock.TotalSeconds;
+
+            double numberOfBlocksDifference = secondsDifference / secondsPerBlock;
+            var blockDifference = Convert.ToUInt64(numberOfBlocksDifference);
+            if (blockDifference < 1) blockDifference = 1;
+            return blockDifference;
+        }
+
+        private void EnsureRecentBlockIfNecessary(DateTime moment, TimeSpan timePerBlock)
+        {
+            var max = entries.Keys.Max();
+            var latest = entries[max];
+            var maxRetry = 10;
+            while (moment > latest.Utc)
+            {
+                var newBlock = AddCurrentBlock();
+                if (newBlock == null || newBlock.BlockNumber == latest.BlockNumber)
+                {
+                    maxRetry--;
+                    if (maxRetry == 0) throw new Exception("Unable to fetch recent block after 10x tries.");
+                    Thread.Sleep(timePerBlock);
+                }
+                max = entries.Keys.Max();
+                latest = entries[max];
+            }
+        }
+
+        private BlockTimeEntry? AddBlockNumber(decimal blockNumber)
+        {
+            return AddBlockNumber(Convert.ToUInt64(blockNumber));
+        }
+
+        private BlockTimeEntry? AddBlockNumber(ulong blockNumber)
+        {
+            if (entries.ContainsKey(blockNumber))
+            {
+                return entries[blockNumber];
+            }
+
+            if (entries.Count > MaxEntries)
+            {
+                log.Debug("Entries cleared!");
+                entries.Clear();
+                Initialize();
+            }
+
+            var time = GetTimestampFromBlock(blockNumber);
+            if (time == null)
+            {
+                log.Log("Failed to get block for number: " + blockNumber);
+                return null;
+            }
+            var entry = new BlockTimeEntry(blockNumber, time.Value);
+            log.Debug("Found block " + entry.BlockNumber + " at " + entry.Utc.ToString("o"));
+            entries.Add(blockNumber, entry);
+            return entry;
+        }
+
+        private TimeSpan EstimateTimePerBlock()
+        {
+            var min = entries.Keys.Min();
+            var max = entries.Keys.Max();
+            var clippedMin = Math.Max(max - 100, min);
+            var minTime = entries[min].Utc;
+            var clippedMinBlock = AddBlockNumber(clippedMin);
+            if (clippedMinBlock != null) minTime = clippedMinBlock.Utc;
+
+            var maxTime = entries[max].Utc;
+            var elapsedTime = maxTime - minTime;
+
+            double elapsedSeconds = elapsedTime.TotalSeconds;
+            double numberOfBlocks = max - min;
+            double secondsPerBlock = elapsedSeconds / numberOfBlocks;
+
+            var result = TimeSpan.FromSeconds(secondsPerBlock);
+            if (result.TotalSeconds < 1.0) result = TimeSpan.FromSeconds(1.0);
+            return result;
+        }
+
+        private void Initialize()
+        {
+            if (!entries.Any())
+            {
+                AddCurrentBlock();
+                AddBlockNumber(entries.Single().Key - 1);
+            }
+        }
+
+        private static void AssertMomentIsInPast(DateTime moment)
+        {
+            if (moment > DateTime.UtcNow) throw new Exception("Moment must be UTC and must be in the past.");
+        }
+
+        private BlockTimeEntry? AddCurrentBlock()
+        {
+            var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
+            var blockNumber = number.ToDecimal();
+            return AddBlockNumber(blockNumber);
+        }
+
+        private DateTime? GetTimestampFromBlock(ulong blockNumber)
+        {
+            try
+            {
+                var block = Time.Wait(web3.Eth.Blocks.GetBlockWithTransactionsByNumber.SendRequestAsync(new BlockParameter(blockNumber)));
+                if (block == null) return null;
+                return DateTimeOffset.FromUnixTimeSeconds(Convert.ToInt64(block.Timestamp.ToDecimal())).UtcDateTime;
+            }
+            catch (Exception ex)
+            {
+                int i = 0;
+                throw;
+            }
+        }
+
+        private BlockTimeEntry? FindClosestBeforeEntry(DateTime moment)
+        {
+            BlockTimeEntry? result = null;
+            foreach (var entry in entries.Values)
+            {
+                if (result == null)
+                {
+                    if (entry.Utc < moment) result = entry;
+                }
+                else
+                {
+                    if (entry.Utc > result.Utc && entry.Utc < moment) result = entry;
+                }
+            }
+            return result;
+        }
+
+        private BlockTimeEntry? FindClosestAfterEntry(DateTime moment)
+        {
+            BlockTimeEntry? result = null;
+            foreach (var entry in entries.Values)
+            {
+                if (result == null)
+                {
+                    if (entry.Utc > moment) result = entry;
+                }
+                else
+                {
+                    if (entry.Utc < result.Utc && entry.Utc > moment) result = entry;
+                }
+            }
+            return result;
+        }
+    }
+}

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -1,7 +1,9 @@
 using Logging;
+using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
+using System.Runtime.CompilerServices;
 using Utils;
 
 namespace NethereumWorkflow
@@ -17,34 +19,47 @@ namespace NethereumWorkflow
             this.web3 = web3;
         }
 
-        public void SendEth(string toAddress, decimal ethAmount)
+        public string SendEth(string toAddress, decimal ethAmount)
         {
+            log.Debug();
             var receipt = Time.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(toAddress, ethAmount));
             if (!receipt.Succeeded()) throw new Exception("Unable to send Eth");
+            return receipt.TransactionHash;
         }
 
         public decimal GetEthBalance()
         {
+            log.Debug();
             return GetEthBalance(web3.TransactionManager.Account.Address);
         }
 
         public decimal GetEthBalance(string address)
         {
+            log.Debug();
             var balance = Time.Wait(web3.Eth.GetBalance.SendRequestAsync(address));
             return Web3.Convert.FromWei(balance.Value);
         }
 
         public TResult Call<TFunction, TResult>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
         {
+            log.Debug(typeof(TFunction).ToString());
             var handler = web3.Eth.GetContractQueryHandler<TFunction>();
             return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function));
         }
 
-        public void SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
+        public string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
         {
+            log.Debug();
             var handler = web3.Eth.GetContractTransactionHandler<TFunction>();
             var receipt = Time.Wait(handler.SendRequestAndWaitForReceiptAsync(contractAddress, function));
             if (!receipt.Succeeded()) throw new Exception("Unable to perform contract transaction.");
+            return receipt.TransactionHash;
+        }
+
+        public Transaction GetTransaction(string transactionHash)
+        {
+            log.Debug();
+            return Time.Wait(web3.Eth.Transactions.GetTransactionByHash.SendRequestAsync(transactionHash));
         }
 
         public decimal? GetSyncedBlockNumber()
@@ -70,5 +85,24 @@ namespace NethereumWorkflow
                 return false;
             }
         }
+
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new()
+        {
+            var blockTimeFinder = new BlockTimeFinder(web3, log);
+
+            var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From);
+            var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To);
+
+            return GetEvents<TEvent>(address, fromBlock, toBlock);
+        }
+
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new()
+        {
+            var eventHandler = web3.Eth.GetEvent<TEvent>(address);
+            var from = new BlockParameter(fromBlockNumber);
+            var to = new BlockParameter(toBlockNumber);
+            var blockFilter = Time.Wait(eventHandler.CreateFilterBlockRangeAsync(from, to));
+            return Time.Wait(eventHandler.GetAllChangesAsync(blockFilter));
+        }
     }
 }

Framework/NethereumWorkflow/NethereumInteractionCreator.cs

@@ -20,6 +20,7 @@ namespace NethereumWorkflow
 
         public NethereumInteraction CreateWorkflow()
         {
+            log.Debug("Starting interaction to " + ip + ":" + port);
             return new NethereumInteraction(log, CreateWeb3());
         }
 

Framework/Utils/Address.cs

@@ -10,5 +10,15 @@
 
         public string Host { get; }
         public int Port { get; }
+
+        public override string ToString()
+        {
+            return $"{Host}:{Port}";
+        }
+
+        public bool IsValid()
+        {
+            return !string.IsNullOrEmpty(Host) && Port > 0;
+        }
     }
 }

Framework/Utils/ByteSize.cs

@@ -46,6 +46,18 @@
         }
     }
 
+    public class BytesPerSecond : ByteSize
+    {
+        public BytesPerSecond(long sizeInBytes) : base(sizeInBytes)
+        {
+        }
+
+        public override string ToString()
+        {
+            return base.ToString() + "/s";
+        }
+    }
+
     public static class ByteSizeIntExtensions
     {
         private const long Kilo = 1024;

Framework/Utils/TaskFactory.cs

@@ -1,15 +1,27 @@
-namespace ContinuousTests
+namespace Utils
 {
     public class TaskFactory
     {
         private readonly object taskLock = new();
         private readonly List<Task> activeTasks = new List<Task>();
 
-        public void Run(Action action)
+        public void Run(Action action, string name)
         {
             lock (taskLock)
             {
-                activeTasks.Add(Task.Run(action).ContinueWith(CleanupTask, null));
+                activeTasks.Add(Task.Run(() => CatchException(action, name)).ContinueWith(CleanupTask, null));
+            }
+        }
+
+        private void CatchException(Action action, string name)
+        {
+            try
+            {
+                action();
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Exception in task '{name}': " + ex);
             }
         }
 

Framework/Utils/Time.cs

@@ -13,6 +13,11 @@
             return task.Result;
         }
 
+        public static void Wait(Task task)
+        {
+            task.Wait();
+        }
+
         public static string FormatDuration(TimeSpan d)
         {
             var result = "";
@@ -23,6 +28,35 @@
             return result;
         }
         
+        public static TimeSpan ParseTimespan(string span)
+        {
+            span = span.Replace(" ", "").Replace(",", "");
+            var result = TimeSpan.Zero;
+            var number = "";
+            foreach (var c in span)
+            {
+                if (char.IsNumber(c)) number += c;
+                else
+                {
+                    var value = Convert.ToInt32(number);
+                    number = "";
+
+                    if (c == 'd') result += TimeSpan.FromDays(value);
+                    else if (c == 'h') result += TimeSpan.FromHours(value);
+                    else if (c == 'm') result += TimeSpan.FromMinutes(value);
+                    else if (c == 's') result += TimeSpan.FromSeconds(value);
+                    else throw new Exception("Unknown time modifier: " + c);
+                }
+            }
+
+            if (!string.IsNullOrEmpty(number))
+            {
+                var value = Convert.ToInt32(number);
+                result += TimeSpan.FromSeconds(value);
+            }
+            return result;
+        }
+
         public static void WaitUntil(Func<bool> predicate)
         {
             WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));
@@ -46,33 +80,35 @@
 
         public static void Retry(Action action, string description)
         {
-            Retry(action, TimeSpan.FromMinutes(1), description);
+            Retry(action, 1, description);
         }
 
         public static T Retry<T>(Func<T> action, string description)
         {
-            return Retry(action, TimeSpan.FromMinutes(1), description);
+            return Retry(action, 1, description);
         }
 
-        public static void Retry(Action action, TimeSpan timeout, string description)
+        public static void Retry(Action action, int maxRetries, string description)
         {
-            Retry(action, timeout, TimeSpan.FromSeconds(1), description);
+            Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
         }
 
-        public static T Retry<T>(Func<T> action, TimeSpan timeout, string description)
+        public static T Retry<T>(Func<T> action, int maxRetries, string description)
         {
-            return Retry(action, timeout, TimeSpan.FromSeconds(1), description);
+            return Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
         }
 
-        public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime, string description)
+        public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)
         {
             var start = DateTime.UtcNow;
+            var retries = 0;
             var exceptions = new List<Exception>();
             while (true)
             {
-                if (DateTime.UtcNow - start > timeout)
+                if (retries > maxRetries)
                 {
-                    throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
+                    var duration = DateTime.UtcNow - start;
+                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
                 }
 
                 try
@@ -83,21 +119,24 @@
                 catch (Exception ex)
                 {
                     exceptions.Add(ex);
+                    retries++;
                 }
 
                 Sleep(retryTime);
             }
         }
 
-        public static T Retry<T>(Func<T> action, TimeSpan timeout, TimeSpan retryTime, string description)
+        public static T Retry<T>(Func<T> action, int maxRetries, TimeSpan retryTime, string description)
         {
             var start = DateTime.UtcNow;
+            var retries = 0;
             var exceptions = new List<Exception>();
             while (true)
             {
-                if (DateTime.UtcNow - start > timeout)
+                if (retries > maxRetries)
                 {
-                    throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
+                    var duration = DateTime.UtcNow - start;
+                    throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
                 }
 
                 try
@@ -107,6 +146,7 @@
                 catch (Exception ex)
                 {
                     exceptions.Add(ex);
+                    retries++;
                 }
 
                 Sleep(retryTime);

Framework/Utils/TimeRange.cs

@@ -0,0 +1,24 @@
+namespace Utils
+{
+    public class TimeRange
+    {
+        public TimeRange(DateTime from, DateTime to)
+        {
+            if (from < to)
+            {
+                From = from;
+                To = to;
+            }
+            else
+            {
+                From = to;
+                To = from;
+            }
+            Duration = To - From;
+        }
+
+        public DateTime From { get; }
+        public DateTime To { get; }
+        public TimeSpan Duration { get; }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -1,5 +1,11 @@
-using GethPlugin;
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
 using Logging;
+using Nethereum.ABI;
+using Nethereum.Hex.HexTypes;
+using Nethereum.Util;
+using NethereumWorkflow;
+using Utils;
 
 namespace CodexContractsPlugin
 {
@@ -7,45 +13,168 @@ namespace CodexContractsPlugin
     {
         CodexContractsDeployment Deployment { get; }
 
-        void MintTestTokens(IGethNode gethNode, IHasEthAddress owner, TestToken testTokens);
+        bool IsDeployed();
-        void MintTestTokens(IGethNode gethNode, EthAddress ethAddress, TestToken testTokens);
+        string MintTestTokens(IHasEthAddress owner, TestToken testTokens);
-        TestToken GetTestTokenBalance(IGethNode gethNode, IHasEthAddress owner);
+        string MintTestTokens(EthAddress ethAddress, TestToken testTokens);
-        TestToken GetTestTokenBalance(IGethNode gethNode, EthAddress ethAddress);
+        TestToken GetTestTokenBalance(IHasEthAddress owner);
+        TestToken GetTestTokenBalance(EthAddress ethAddress);
+
+        Request[] GetStorageRequests(TimeRange timeRange);
+        EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
+        RequestState GetRequestState(Request request);
+        RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange);
+        RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange);
+        SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange);
+        SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange);
+    }
+
+    public enum RequestState
+    {
+        New,
+        Started,
+        Cancelled,
+        Finished,
+        Failed
     }
 
     public class CodexContractsAccess : ICodexContracts
     {
         private readonly ILog log;
+        private readonly IGethNode gethNode;
 
-        public CodexContractsAccess(ILog log, CodexContractsDeployment deployment)
+        public CodexContractsAccess(ILog log, IGethNode gethNode, CodexContractsDeployment deployment)
         {
             this.log = log;
+            this.gethNode = gethNode;
             Deployment = deployment;
         }
 
         public CodexContractsDeployment Deployment { get; }
 
-        public void MintTestTokens(IGethNode gethNode, IHasEthAddress owner, TestToken testTokens)
+        public bool IsDeployed()
         {
-            MintTestTokens(gethNode, owner.EthAddress, testTokens);
+            return !string.IsNullOrEmpty(StartInteraction().GetTokenName(Deployment.TokenAddress));
         }
 
-        public void MintTestTokens(IGethNode gethNode, EthAddress ethAddress, TestToken testTokens)
+        public string MintTestTokens(IHasEthAddress owner, TestToken testTokens)
         {
-            var interaction = new ContractInteractions(log, gethNode);
+            return MintTestTokens(owner.EthAddress, testTokens);
-            interaction.MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
         }
 
-        public TestToken GetTestTokenBalance(IGethNode gethNode, IHasEthAddress owner)
+        public string MintTestTokens(EthAddress ethAddress, TestToken testTokens)
         {
-            return GetTestTokenBalance(gethNode, owner.EthAddress);
+            return StartInteraction().MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
         }
 
-        public TestToken GetTestTokenBalance(IGethNode gethNode, EthAddress ethAddress)
+        public TestToken GetTestTokenBalance(IHasEthAddress owner)
         {
-            var interaction = new ContractInteractions(log, gethNode);
+            return GetTestTokenBalance(owner.EthAddress);
-            var balance = interaction.GetBalance(Deployment.TokenAddress, ethAddress.Address);
+        }
+
+        public TestToken GetTestTokenBalance(EthAddress ethAddress)
+        {
+            var balance = StartInteraction().GetBalance(Deployment.TokenAddress, ethAddress.Address);
             return balance.TestTokens();
         }
+
+        public Request[] GetStorageRequests(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<StorageRequestedEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            var i = StartInteraction();
+            return events
+                    .Select(e =>
+                    {
+                        var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId);
+                        var result = requestEvent.ReturnValue1;
+                        result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                        result.RequestId = e.Event.RequestId;
+                        return result;
+                    })
+                    .ToArray();
+        }
+
+        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<RequestCancelledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<SlotFilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<SlotFreedEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
+        {
+            var encoder = new ABIEncode();
+            var encoded = encoder.GetABIEncoded(
+                new ABIValue("bytes32", storageRequest.RequestId),
+                new ABIValue("uint256", slotIndex.ToBig())
+            );
+
+            var hashed = Sha3Keccack.Current.CalculateHash(encoded);
+
+            var func = new GetHostFunction
+            {
+                SlotId = hashed
+            };
+            var address = gethNode.Call<GetHostFunction, string>(Deployment.MarketplaceAddress, func);
+            if (string.IsNullOrEmpty(address)) return null;
+            return new EthAddress(address);
+        }
+
+        public RequestState GetRequestState(Request request)
+        {
+            var func = new RequestStateFunction
+            {
+                RequestId = request.RequestId
+            };
+            return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
+        }
+
+        private EthAddress GetEthAddressFromTransaction(string transactionHash)
+        {
+            var transaction = gethNode.GetTransaction(transactionHash);
+            return new EthAddress(transaction.From);
+        }
+
+        private ContractInteractions StartInteraction()
+        {
+            return new ContractInteractions(log, gethNode);
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -1,10 +1,13 @@
-using KubernetesWorkflow;
+using GethPlugin;
+using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
+using Logging;
 
 namespace CodexContractsPlugin
 {
     public class CodexContractsContainerRecipe : ContainerRecipeFactory
     {
-        public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:latest-dist-tests";
+        public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests";
 
         public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
         public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
@@ -16,10 +19,11 @@ namespace CodexContractsPlugin
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 
-            var ip = config.GethNode.StartResult.Container.Pod.PodInfo.Ip;
+            var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
-            var port = config.GethNode.StartResult.HttpPort.Number;
 
-            AddEnvVar("DISTTEST_NETWORK_URL", $"http://{ip}:{port}");
+            SetSchedulingAffinity(notIn: "false");
+
+            AddEnvVar("DISTTEST_NETWORK_URL", address.ToString());
             AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork");
             AddEnvVar("KEEP_ALIVE", "1");
         }

ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.cs

@@ -35,10 +35,10 @@ namespace CodexContractsPlugin
             return starter.Deploy(ci, gethNode);
         }
 
-        public ICodexContracts WrapDeploy(CodexContractsDeployment deployment)
+        public ICodexContracts WrapDeploy(IGethNode gethNode, CodexContractsDeployment deployment)
         {
             deployment = SerializeGate.Gate(deployment);
-            return starter.Wrap(deployment);
+            return starter.Wrap(gethNode, deployment);
         }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -1,6 +1,7 @@
 using Core;
 using GethPlugin;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using Utils;
 
@@ -31,20 +32,23 @@ namespace CodexContractsPlugin
 
             try
             {
-                return DeployContract(container, workflow, gethNode);
+                var result = DeployContract(container, workflow, gethNode);
+                workflow.Stop(containers);
+                Log("Container stopped.");
+                return result;
             }
-            catch
+            catch (Exception ex)
             {
-                Log("Failed to deploy contract.");
+                Log("Failed to deploy contract: " + ex);
                 Log("Downloading Codex SmartContracts container log...");
                 ci.DownloadLog(container);
                 throw;
             }
         }
 
-        public ICodexContracts Wrap(CodexContractsDeployment deployment)
+        public ICodexContracts Wrap(IGethNode gethNode, CodexContractsDeployment deployment)
         {
-            return new CodexContractsAccess(tools.GetLog(), deployment);
+            return new CodexContractsAccess(tools.GetLog(), gethNode, deployment);
         }
 
         private CodexContractsDeployment DeployContract(RunningContainer container, IStartupWorkflow workflow, IGethNode gethNode)
@@ -81,7 +85,7 @@ namespace CodexContractsPlugin
 
         private void WaitUntil(Func<bool> predicate)
         {
-            Time.WaitUntil(predicate, TimeSpan.FromMinutes(3), TimeSpan.FromSeconds(2));
+            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2));
         }
 
         private StartupConfig CreateStartupConfig(IGethNode gethNode)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -1,7 +1,9 @@
-using GethPlugin;
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
 using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
+using Nethereum.Hex.HexConvertors.Extensions;
 using NethereumWorkflow;
 using System.Numerics;
 
@@ -26,9 +28,26 @@ namespace CodexContractsPlugin
             return gethNode.Call<GetTokenFunction, string>(marketplaceAddress, function);
         }
 
-        public void MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
+        public string GetTokenName(string tokenAddress)
         {
-            MintTokens(address.Address, amount, tokenAddress);
+            try
+            {
+                log.Debug(tokenAddress);
+                var function = new GetTokenNameFunction();
+
+                return gethNode.Call<GetTokenNameFunction, string>(tokenAddress, function);
+            }
+            catch (Exception ex)
+            {
+                log.Log("Failed to get token name: " + ex);
+                return string.Empty;
+            }
+        }
+
+        public string MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
+        {
+            log.Debug($"{amount} -> {address} (token: {tokenAddress})");
+            return MintTokens(address.Address, amount, tokenAddress);
         }
 
         public decimal GetBalance(string tokenAddress, string account)
@@ -42,8 +61,20 @@ namespace CodexContractsPlugin
             return gethNode.Call<GetTokenBalanceFunction, BigInteger>(tokenAddress, function).ToDecimal();
         }
 
+        public GetRequestOutputDTO GetRequest(string marketplaceAddress, byte[] requestId)
+        {
+
+            log.Debug($"({marketplaceAddress}) {requestId.ToHex(true)}");
+            var func = new GetRequestFunction
+            {
+                RequestId = requestId
+            };
+            return gethNode.Call<GetRequestFunction, GetRequestOutputDTO>(marketplaceAddress, func);
+        }
+
         public bool IsSynced(string marketplaceAddress, string marketplaceAbi)
         {
+            log.Debug();
             try
             {
                 return IsBlockNumberOK() && IsContractAvailable(marketplaceAddress, marketplaceAbi);
@@ -54,7 +85,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        private void MintTokens(string account, decimal amount, string tokenAddress)
+        private string MintTokens(string account, decimal amount, string tokenAddress)
         {
             log.Debug($"({tokenAddress}) {amount} --> {account}");
             if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
@@ -65,7 +96,7 @@ namespace CodexContractsPlugin
                 Amount = amount.ToBig()
             };
 
-            gethNode.SendTransaction(tokenAddress, function);
+            return gethNode.SendTransaction(tokenAddress, function);
         }
 
         private bool IsBlockNumberOK()
@@ -85,6 +116,11 @@ namespace CodexContractsPlugin
     {
     }
 
+    [Function("name", "string")]
+    public class GetTokenNameFunction : FunctionMessage
+    {
+    }
+
     [Function("mint")]
     public class MintTokensFunction : FunctionMessage
     {

ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
@@ -25,6 +26,7 @@ namespace CodexContractsPlugin
             var marketplaceAddress = Retry(FetchMarketplaceAddress);
             if (string.IsNullOrEmpty(marketplaceAddress)) throw new InvalidOperationException("Unable to fetch marketplace account from codex-contracts node. Test infra failure.");
 
+            log.Debug("Got MarketplaceAddress: " + marketplaceAddress);
             return marketplaceAddress;
         }
 
@@ -34,6 +36,7 @@ namespace CodexContractsPlugin
             var marketplaceAbi = Retry(FetchMarketplaceAbi);
             if (string.IsNullOrEmpty(marketplaceAbi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
 
+            log.Debug("Got Marketplace ABI: " + marketplaceAbi);
             return marketplaceAbi;
         }
 

ProjectPlugins/CodexContractsPlugin/CoreInterfaceExtensions.cs

@@ -10,15 +10,15 @@ namespace CodexContractsPlugin
             return Plugin(ci).DeployContracts(ci, gethNode);
         }
 
-        public static ICodexContracts WrapCodexContractsDeployment(this CoreInterface ci, CodexContractsDeployment deployment)
+        public static ICodexContracts WrapCodexContractsDeployment(this CoreInterface ci, IGethNode gethNode, CodexContractsDeployment deployment)
         {
-            return Plugin(ci).WrapDeploy(deployment);
+            return Plugin(ci).WrapDeploy(gethNode, deployment);
         }
 
         public static ICodexContracts StartCodexContracts(this CoreInterface ci, IGethNode gethNode)
         {
             var deployment = DeployCodexContracts(ci, gethNode);
-            return WrapCodexContractsDeployment(ci, deployment);
+            return WrapCodexContractsDeployment(ci, gethNode, deployment);
         }
 
         private static CodexContractsPlugin Plugin(CoreInterface ci)

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -0,0 +1,35 @@
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
+using GethPlugin;
+
+namespace CodexContractsPlugin.Marketplace
+{
+    public partial class Request : RequestBase
+    {
+        public ulong BlockNumber { get; set; }
+        public byte[] RequestId { get; set; }
+
+        public EthAddress ClientAddress { get { return new EthAddress(Client); } }
+    }
+
+    public partial class RequestFulfilledEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+    }
+
+    public partial class RequestCancelledEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+    }
+
+    public partial class SlotFilledEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+        public EthAddress Host { get; set; }
+    }
+
+    public partial class SlotFreedEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+    }
+}
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexContractsPlugin/Marketplace/Marketplace.cs

@@ -0,0 +1,517 @@
+using Nethereum.ABI.FunctionEncoding.Attributes;
+using Nethereum.Contracts;
+using System.Numerics;
+
+// Generated code, do not modify.
+
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
+namespace CodexContractsPlugin.Marketplace
+{
+    public partial class ConfigFunction : ConfigFunctionBase { }
+
+    [Function("config", typeof(ConfigOutputDTO))]
+    public class ConfigFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class FillSlotFunction : FillSlotFunctionBase { }
+
+    [Function("fillSlot")]
+    public class FillSlotFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("uint256", "slotIndex", 2)]
+        public virtual BigInteger SlotIndex { get; set; }
+        [Parameter("bytes", "proof", 3)]
+        public virtual byte[] Proof { get; set; }
+    }
+
+    public partial class FreeSlotFunction : FreeSlotFunctionBase { }
+
+    [Function("freeSlot")]
+    public class FreeSlotFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class GetActiveSlotFunction : GetActiveSlotFunctionBase { }
+
+    [Function("getActiveSlot", typeof(GetActiveSlotOutputDTO))]
+    public class GetActiveSlotFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class GetChallengeFunction : GetChallengeFunctionBase { }
+
+    [Function("getChallenge", "bytes32")]
+    public class GetChallengeFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class GetHostFunction : GetHostFunctionBase { }
+
+    [Function("getHost", "address")]
+    public class GetHostFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class GetPointerFunction : GetPointerFunctionBase { }
+
+    [Function("getPointer", "uint8")]
+    public class GetPointerFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class GetRequestFunction : GetRequestFunctionBase { }
+
+    [Function("getRequest", typeof(GetRequestOutputDTO))]
+    public class GetRequestFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class IsProofRequiredFunction : IsProofRequiredFunctionBase { }
+
+    [Function("isProofRequired", "bool")]
+    public class IsProofRequiredFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class MarkProofAsMissingFunction : MarkProofAsMissingFunctionBase { }
+
+    [Function("markProofAsMissing")]
+    public class MarkProofAsMissingFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+        [Parameter("uint256", "period", 2)]
+        public virtual BigInteger Period { get; set; }
+    }
+
+    public partial class MissingProofsFunction : MissingProofsFunctionBase { }
+
+    [Function("missingProofs", "uint256")]
+    public class MissingProofsFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class MyRequestsFunction : MyRequestsFunctionBase { }
+
+    [Function("myRequests", "bytes32[]")]
+    public class MyRequestsFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class MySlotsFunction : MySlotsFunctionBase { }
+
+    [Function("mySlots", "bytes32[]")]
+    public class MySlotsFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class RequestEndFunction : RequestEndFunctionBase { }
+
+    [Function("requestEnd", "uint256")]
+    public class RequestEndFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestStateFunction : RequestStateFunctionBase { }
+
+    [Function("requestState", "uint8")]
+    public class RequestStateFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestStorageFunction : RequestStorageFunctionBase { }
+
+    [Function("requestStorage")]
+    public class RequestStorageFunctionBase : FunctionMessage
+    {
+        [Parameter("tuple", "request", 1)]
+        public virtual Request Request { get; set; }
+    }
+
+    public partial class SlotStateFunction : SlotStateFunctionBase { }
+
+    [Function("slotState", "uint8")]
+    public class SlotStateFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class SubmitProofFunction : SubmitProofFunctionBase { }
+
+    [Function("submitProof")]
+    public class SubmitProofFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+        [Parameter("bytes", "proof", 2)]
+        public virtual byte[] Proof { get; set; }
+    }
+
+    public partial class TokenFunction : TokenFunctionBase { }
+
+    [Function("token", "address")]
+    public class TokenFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class WillProofBeRequiredFunction : WillProofBeRequiredFunctionBase { }
+
+    [Function("willProofBeRequired", "bool")]
+    public class WillProofBeRequiredFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class WithdrawFundsFunction : WithdrawFundsFunctionBase { }
+
+    [Function("withdrawFunds")]
+    public class WithdrawFundsFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class ProofSubmittedEventDTO : ProofSubmittedEventDTOBase { }
+
+    [Event("ProofSubmitted")]
+    public class ProofSubmittedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "id", 1, false)]
+        public virtual byte[] Id { get; set; }
+        [Parameter("bytes", "proof", 2, false)]
+        public virtual byte[] Proof { get; set; }
+    }
+
+    public partial class RequestCancelledEventDTO : RequestCancelledEventDTOBase { }
+
+    [Event("RequestCancelled")]
+    public class RequestCancelledEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestFailedEventDTO : RequestFailedEventDTOBase { }
+
+    [Event("RequestFailed")]
+    public class RequestFailedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestFulfilledEventDTO : RequestFulfilledEventDTOBase { }
+
+    [Event("RequestFulfilled")]
+    public class RequestFulfilledEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class SlotFilledEventDTO : SlotFilledEventDTOBase { }
+
+    [Event("SlotFilled")]
+    public class SlotFilledEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("uint256", "slotIndex", 2, false)]
+        public virtual BigInteger SlotIndex { get; set; }
+    }
+
+    public partial class SlotFreedEventDTO : SlotFreedEventDTOBase { }
+
+    [Event("SlotFreed")]
+    public class SlotFreedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("uint256", "slotIndex", 2, false)]
+        public virtual BigInteger SlotIndex { get; set; }
+    }
+
+    public partial class StorageRequestedEventDTO : StorageRequestedEventDTOBase { }
+
+    [Event("StorageRequested")]
+    public class StorageRequestedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, false)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("tuple", "ask", 2, false)]
+        public virtual Ask Ask { get; set; }
+        [Parameter("uint256", "expiry", 3, false)]
+        public virtual BigInteger Expiry { get; set; }
+    }
+
+    public partial class ConfigOutputDTO : ConfigOutputDTOBase { }
+
+    [FunctionOutput]
+    public class ConfigOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("tuple", "collateral", 1)]
+        public virtual CollateralConfig Collateral { get; set; }
+        [Parameter("tuple", "proofs", 2)]
+        public virtual ProofConfig Proofs { get; set; }
+    }
+
+
+
+
+
+    public partial class GetActiveSlotOutputDTO : GetActiveSlotOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetActiveSlotOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("tuple", "", 1)]
+        public virtual ActiveSlot ReturnValue1 { get; set; }
+    }
+
+    public partial class GetChallengeOutputDTO : GetChallengeOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetChallengeOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bytes32", "", 1)]
+        public virtual byte[] ReturnValue1 { get; set; }
+    }
+
+    public partial class GetHostOutputDTO : GetHostOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetHostOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("address", "", 1)]
+        public virtual string ReturnValue1 { get; set; }
+    }
+
+    public partial class GetPointerOutputDTO : GetPointerOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetPointerOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint8", "", 1)]
+        public virtual byte ReturnValue1 { get; set; }
+    }
+
+    public partial class GetRequestOutputDTO : GetRequestOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetRequestOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("tuple", "", 1)]
+        public virtual Request ReturnValue1 { get; set; }
+    }
+
+    public partial class IsProofRequiredOutputDTO : IsProofRequiredOutputDTOBase { }
+
+    [FunctionOutput]
+    public class IsProofRequiredOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bool", "", 1)]
+        public virtual bool ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class MissingProofsOutputDTO : MissingProofsOutputDTOBase { }
+
+    [FunctionOutput]
+    public class MissingProofsOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint256", "", 1)]
+        public virtual BigInteger ReturnValue1 { get; set; }
+    }
+
+    public partial class MyRequestsOutputDTO : MyRequestsOutputDTOBase { }
+
+    [FunctionOutput]
+    public class MyRequestsOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bytes32[]", "", 1)]
+        public virtual List<byte[]> ReturnValue1 { get; set; }
+    }
+
+    public partial class MySlotsOutputDTO : MySlotsOutputDTOBase { }
+
+    [FunctionOutput]
+    public class MySlotsOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bytes32[]", "", 1)]
+        public virtual List<byte[]> ReturnValue1 { get; set; }
+    }
+
+    public partial class RequestEndOutputDTO : RequestEndOutputDTOBase { }
+
+    [FunctionOutput]
+    public class RequestEndOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint256", "", 1)]
+        public virtual BigInteger ReturnValue1 { get; set; }
+    }
+
+    public partial class RequestStateOutputDTO : RequestStateOutputDTOBase { }
+
+    [FunctionOutput]
+    public class RequestStateOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint8", "", 1)]
+        public virtual byte ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class SlotStateOutputDTO : SlotStateOutputDTOBase { }
+
+    [FunctionOutput]
+    public class SlotStateOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint8", "", 1)]
+        public virtual byte ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class TokenOutputDTO : TokenOutputDTOBase { }
+
+    [FunctionOutput]
+    public class TokenOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("address", "", 1)]
+        public virtual string ReturnValue1 { get; set; }
+    }
+
+    public partial class WillProofBeRequiredOutputDTO : WillProofBeRequiredOutputDTOBase { }
+
+    [FunctionOutput]
+    public class WillProofBeRequiredOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bool", "", 1)]
+        public virtual bool ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class CollateralConfig : CollateralConfigBase { }
+
+    public class CollateralConfigBase
+    {
+        [Parameter("uint8", "repairRewardPercentage", 1)]
+        public virtual byte RepairRewardPercentage { get; set; }
+        [Parameter("uint8", "maxNumberOfSlashes", 2)]
+        public virtual byte MaxNumberOfSlashes { get; set; }
+        [Parameter("uint16", "slashCriterion", 3)]
+        public virtual ushort SlashCriterion { get; set; }
+        [Parameter("uint8", "slashPercentage", 4)]
+        public virtual byte SlashPercentage { get; set; }
+    }
+
+    public partial class ProofConfig : ProofConfigBase { }
+
+    public class ProofConfigBase
+    {
+        [Parameter("uint256", "period", 1)]
+        public virtual BigInteger Period { get; set; }
+        [Parameter("uint256", "timeout", 2)]
+        public virtual BigInteger Timeout { get; set; }
+        [Parameter("uint8", "downtime", 3)]
+        public virtual byte Downtime { get; set; }
+    }
+
+    public partial class MarketplaceConfig : MarketplaceConfigBase { }
+
+    public class MarketplaceConfigBase
+    {
+        [Parameter("tuple", "collateral", 1)]
+        public virtual CollateralConfig Collateral { get; set; }
+        [Parameter("tuple", "proofs", 2)]
+        public virtual ProofConfig Proofs { get; set; }
+    }
+
+    public partial class Ask : AskBase { }
+
+    public class AskBase
+    {
+        [Parameter("uint64", "slots", 1)]
+        public virtual ulong Slots { get; set; }
+        [Parameter("uint256", "slotSize", 2)]
+        public virtual BigInteger SlotSize { get; set; }
+        [Parameter("uint256", "duration", 3)]
+        public virtual BigInteger Duration { get; set; }
+        [Parameter("uint256", "proofProbability", 4)]
+        public virtual BigInteger ProofProbability { get; set; }
+        [Parameter("uint256", "reward", 5)]
+        public virtual BigInteger Reward { get; set; }
+        [Parameter("uint256", "collateral", 6)]
+        public virtual BigInteger Collateral { get; set; }
+        [Parameter("uint64", "maxSlotLoss", 7)]
+        public virtual ulong MaxSlotLoss { get; set; }
+    }
+
+    public partial class Content : ContentBase { }
+
+    public class ContentBase
+    {
+        [Parameter("string", "cid", 1)]
+        public virtual string Cid { get; set; }
+        [Parameter("bytes32", "merkleRoot", 2)]
+        public virtual byte[] MerkleRoot { get; set; }
+    }
+
+    public partial class Request : RequestBase { }
+
+    public class RequestBase
+    {
+        [Parameter("address", "client", 1)]
+        public virtual string Client { get; set; }
+        [Parameter("tuple", "ask", 2)]
+        public virtual Ask Ask { get; set; }
+        [Parameter("tuple", "content", 3)]
+        public virtual Content Content { get; set; }
+        [Parameter("uint256", "expiry", 4)]
+        public virtual BigInteger Expiry { get; set; }
+        [Parameter("bytes32", "nonce", 5)]
+        public virtual byte[] Nonce { get; set; }
+    }
+
+    public partial class ActiveSlot : ActiveSlotBase { }
+
+    public class ActiveSlotBase
+    {
+        [Parameter("tuple", "request", 1)]
+        public virtual Request Request { get; set; }
+        [Parameter("uint256", "slotIndex", 2)]
+        public virtual BigInteger SlotIndex { get; set; }
+    }
+}
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexContractsPlugin/Marketplace/README.md

@@ -0,0 +1 @@
+This code was generated using the Nethereum code generator, here: http://playground.nethereum.com

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs

@@ -0,0 +1,46 @@
+using Core;
+using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
+
+namespace CodexDiscordBotPlugin
+{
+    public class CodexDiscordBotPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata
+    {
+        private readonly IPluginTools tools;
+
+        public CodexDiscordBotPlugin(IPluginTools tools)
+        {
+            this.tools = tools;
+        }
+
+        public string LogPrefix => "(DiscordBot) ";
+
+        public void Announce()
+        {
+            tools.GetLog().Log($"Codex DiscordBot (BiblioTech) loaded.");
+        }
+
+        public void AddMetadata(IAddMetadata metadata)
+        {
+            metadata.Add("codexdiscordbotid", new DiscordBotContainerRecipe().Image);
+        }
+
+        public void Decommission()
+        {
+        }
+
+        public RunningContainers Deploy(DiscordBotStartupConfig config)
+        {
+            var workflow = tools.CreateWorkflow();
+            return StartContainer(workflow, config);
+        }
+
+        private RunningContainers StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config)
+        {
+            var startupConfig = new StartupConfig();
+            startupConfig.NameOverride = config.Name;
+            startupConfig.Add(config);
+            return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig);
+        }
+    }
+}

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
+    <ProjectReference Include="..\CodexPlugin\CodexPlugin.csproj" />
+  </ItemGroup>
+
+</Project>

ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs

@@ -0,0 +1,18 @@
+using Core;
+using KubernetesWorkflow.Types;
+
+namespace CodexDiscordBotPlugin
+{
+    public static class CoreInterfaceExtensions
+    {
+        public static RunningContainers DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config)
+        {
+            return Plugin(ci).Deploy(config);
+        }
+
+        private static CodexDiscordBotPlugin Plugin(CoreInterface ci)
+        {
+            return ci.GetPlugin<CodexDiscordBotPlugin>();
+        }
+    }
+}

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs

@@ -0,0 +1,42 @@
+using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
+using Utils;
+
+namespace CodexDiscordBotPlugin
+{
+    public class DiscordBotContainerRecipe : ContainerRecipeFactory
+    {
+        public override string AppName => "discordbot-bibliotech";
+        public override string Image => "thatbenbierens/codex-discordbot:initial";
+
+        protected override void Initialize(StartupConfig startupConfig)
+        {
+            var config = startupConfig.Get<DiscordBotStartupConfig>();
+
+            SetSchedulingAffinity(notIn: "false");
+
+            AddEnvVar("TOKEN", config.Token);
+            AddEnvVar("SERVERNAME", config.ServerName);
+            AddEnvVar("ADMINROLE", config.AdminRoleName);
+            AddEnvVar("ADMINCHANNELNAME", config.AdminChannelName);
+            AddEnvVar("KUBECONFIG", "/opt/kubeconfig.yaml");
+            AddEnvVar("KUBENAMESPACE", config.KubeNamespace);
+
+            var gethInfo = config.GethInfo;
+            AddEnvVar("GETH_HOST", gethInfo.Host);
+            AddEnvVar("GETH_HTTP_PORT", gethInfo.Port.ToString());
+            AddEnvVar("GETH_PRIVATE_KEY", gethInfo.PrivKey);
+            AddEnvVar("CODEXCONTRACTS_MARKETPLACEADDRESS", gethInfo.MarketplaceAddress);
+            AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress);
+            AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi);
+
+            if (!string.IsNullOrEmpty(config.DataPath))
+            {
+                AddEnvVar("DATAPATH", config.DataPath);
+                AddVolume(config.DataPath, 1.GB());
+            }
+
+            AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml", secret: "discordbot-sa-kubeconfig");
+        }
+    }
+}

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs

@@ -0,0 +1,45 @@
+namespace CodexDiscordBotPlugin
+{
+    public class DiscordBotStartupConfig
+    {
+        public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName, string kubeNamespace, DiscordBotGethInfo gethInfo)
+        {
+            Name = name;
+            Token = token;
+            ServerName = serverName;
+            AdminRoleName = adminRoleName;
+            AdminChannelName = adminChannelName;
+            KubeNamespace = kubeNamespace;
+            GethInfo = gethInfo;
+        }
+
+        public string Name { get; }
+        public string Token { get; }
+        public string ServerName { get; }
+        public string AdminRoleName { get; }
+        public string AdminChannelName { get; }
+        public string KubeNamespace { get; }
+        public DiscordBotGethInfo GethInfo { get; }
+        public string? DataPath { get; set; }
+    }
+
+    public class DiscordBotGethInfo
+    {
+        public DiscordBotGethInfo(string host, int port, string privKey, string marketplaceAddress, string tokenAddress, string abi)
+        {
+            Host = host;
+            Port = port;
+            PrivKey = privKey;
+            MarketplaceAddress = marketplaceAddress;
+            TokenAddress = tokenAddress;
+            Abi = abi;
+        }
+
+        public string Host { get; }
+        public int Port { get; }
+        public string PrivKey { get; }
+        public string MarketplaceAddress { get; }
+        public string TokenAddress { get; }
+        public string Abi { get; }
+    }
+}

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -1,5 +1,7 @@
 using Core;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
+using Utils;
 
 namespace CodexPlugin
 {
@@ -39,7 +41,7 @@ namespace CodexPlugin
                 };
             }
 
-            var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
+            var result = http.Deserialize<CodexDebugPeerResponse>(str);
             result.IsPeerFound = true;
             return result;
         }
@@ -49,6 +51,11 @@ namespace CodexPlugin
             return Http().HttpGetJson<CodexDebugBlockExchangeResponse>("debug/blockexchange");
         }
 
+        public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
+        {
+            return LongHttp().HttpGetJson<CodexDebugRepoStoreResponse[]>("debug/repostore");
+        }
+
         public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
         {
             return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");
@@ -56,14 +63,17 @@ namespace CodexPlugin
 
         public string UploadFile(FileStream fileStream)
         {
-            // private const string UploadFailedMessage = "Unable to store block";
+            return Http().HttpPostStream("data", fileStream);
-
-            return Http().HttpPostStream("upload", fileStream);
         }
 
         public Stream DownloadFile(string contentId)
         {
-            return Http().HttpGetStream("download/" + contentId);
+            return Http().HttpGetStream("data/" + contentId + "/network");
+        }
+
+        public CodexLocalDataResponse[] LocalFiles()
+        {
+            return Http().HttpGetJson<CodexLocalDataResponse[]>("data");
         }
 
         public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
@@ -91,9 +101,25 @@ namespace CodexPlugin
             return Container.Name;
         }
 
+        public PodInfo GetPodInfo()
+        {
+            var workflow = tools.CreateWorkflow();
+            return workflow.GetPodInfo(Container);
+        }
+
         private IHttp Http()
         {
-            return tools.CreateHttp(Container.Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
+            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
+        }
+
+        private IHttp LongHttp()
+        {
+            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name);
+        }
+
+        private Address GetAddress()
+        {
+            return Container.GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
         }
 
         private void CheckContainerCrashed(HttpClient client)
@@ -106,6 +132,7 @@ namespace CodexPlugin
             var log = tools.GetLog();
             var file = log.CreateSubfile();
             log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
+            file.Write($"Container Crash Log for {Container.Name}.");
 
             using var reader = new StreamReader(crashLog);
             var line = reader.ReadLine();

ProjectPlugins/CodexPlugin/CodexApiTypes.cs

@@ -5,9 +5,10 @@ namespace CodexPlugin
     public class CodexDebugResponse
     {
         public string id { get; set; } = string.Empty;
-        public string[] addrs { get; set; } = new string[0];
+        public string[] addrs { get; set; } = Array.Empty<string>();
         public string repo { get; set; } = string.Empty;
         public string spr { get; set; } = string.Empty;
+        public string[] announceAddresses { get; set; } = Array.Empty<string>();
         public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
         public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
         public CodexDebugVersionResponse codex { get; set; } = new();
@@ -76,12 +77,7 @@ namespace CodexPlugin
 
         public string peerId { get; set; } = string.Empty;
         public long seqNo { get; set; }
-        public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
+        public string[] addresses { get; set; } = Array.Empty<string>();
-    }
-
-    public class CodexDebugPeerAddressResponse
-    {
-        public string address { get; set; } = string.Empty;
     }
 
     public class CodexDebugThresholdBreaches
@@ -165,4 +161,35 @@ namespace CodexPlugin
         public string wantType { get; set; } = string.Empty;
         public bool sendDontHave { get; set; }
     }
+
+    public class CodexDebugRepoStoreResponse
+    {
+        public string cid { get; set; } = string.Empty; 
+    }
+
+    public class CodexLocalData
+    {
+        public CodexLocalData(ContentId cid, CodexLocalDataManifestResponse manifest)
+        {
+            Cid = cid;
+            Manifest = manifest;
+        }
+
+        public ContentId Cid { get; }
+        public CodexLocalDataManifestResponse Manifest { get; }
+    }
+
+    public class CodexLocalDataResponse
+    {
+        public string cid { get; set; } = string.Empty;
+        public CodexLocalDataManifestResponse manifest { get; set; } = new();
+    }
+
+    public class CodexLocalDataManifestResponse
+    {
+        public string rootHash { get; set; } = string.Empty;
+        public int originalBytes { get; set; }
+        public int blockSize { get; set; }
+        public bool @protected { get; set; }
+    }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -1,4 +1,6 @@
-using KubernetesWorkflow;
+using GethPlugin;
+using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
 using Utils;
 
 namespace CodexPlugin
@@ -8,8 +10,10 @@ namespace CodexPlugin
         private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter();
 
         private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
-        public const string MetricsPortTag = "metrics_port";
+        public const string ApiPortTag = "codex_api_port";
-        public const string DiscoveryPortTag = "discovery-port";
+        public const string ListenPortTag = "codex_listen_port";
+        public const string MetricsPortTag = "codex_metrics_port";
+        public const string DiscoveryPortTag = "codex_discovery_port";
 
         // Used by tests for time-constraint assertions.
         public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
@@ -23,24 +27,38 @@ namespace CodexPlugin
         protected override void Initialize(StartupConfig startupConfig)
         {
             SetResourcesRequest(milliCPUs: 100, memory: 100.MB());
-            SetResourceLimits(milliCPUs: 4000, memory: 12.GB());
+            //SetResourceLimits(milliCPUs: 4000, memory: 12.GB());
+
+            SetSchedulingAffinity(notIn: "false");
+            SetSystemCriticalPriority();
 
             var config = startupConfig.Get<CodexStartupConfig>();
 
-            AddExposedPortAndVar("CODEX_API_PORT");
+            var apiPort = CreateApiPort(config, ApiPortTag);
+            AddEnvVar("CODEX_API_PORT", apiPort);
             AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
 
             var dataDir = $"datadir{ContainerNumber}";
             AddEnvVar("CODEX_DATA_DIR", dataDir);
             AddVolume($"codex/{dataDir}", GetVolumeCapacity(config));
 
-            AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
+            var discPort = CreateDiscoveryPort(config);
+            AddEnvVar("CODEX_DISC_PORT", discPort);
             AddEnvVar("CODEX_LOG_LEVEL", config.LogLevelWithTopics());
 
-            // This makes the node announce itself to its local (pod) IP address.
+            if (config.PublicTestNet != null)
-            AddEnvVar("NAT_IP_AUTO", "true");
+            {
+                // This makes the node announce itself to its public IP address.
+                AddEnvVar("NAT_IP_AUTO", "false");
+                AddEnvVar("NAT_PUBLIC_IP_AUTO", PublicIpService.Address);
+            }
+            else
+            {
+                // This makes the node announce itself to its local (pod) IP address.
+                AddEnvVar("NAT_IP_AUTO", "true");
+            }
 
-            var listenPort = AddInternalPort();
+            var listenPort = CreateListenPort(config);
             AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
 
             if (!string.IsNullOrEmpty(config.BootstrapSpr))
@@ -65,7 +83,7 @@ namespace CodexPlugin
             }
             if (config.MetricsEnabled)
             {
-                var metricsPort = AddInternalPort(MetricsPortTag);
+                var metricsPort = CreateApiPort(config, MetricsPortTag);
                 AddEnvVar("CODEX_METRICS", "true");
                 AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
                 AddEnvVar("CODEX_METRICS_PORT", metricsPort);
@@ -82,11 +100,10 @@ namespace CodexPlugin
             {
                 var mconfig = config.MarketplaceConfig;
                 var gethStart = mconfig.GethNode.StartResult;
-                var ip = gethStart.Container.Pod.PodInfo.Ip;
+                var wsAddress = gethStart.Container.GetInternalAddress(GethContainerRecipe.WsPortTag);
-                var port = gethStart.WsPort.Number;
                 var marketplaceAddress = mconfig.CodexContracts.Deployment.MarketplaceAddress;
 
-                AddEnvVar("CODEX_ETH_PROVIDER", $"ws://{ip}:{port}");
+                AddEnvVar("CODEX_ETH_PROVIDER", $"{wsAddress.Host.Replace("http://", "ws://")}:{wsAddress.Port}");
                 AddEnvVar("CODEX_MARKETPLACE_ADDRESS", marketplaceAddress);
                 AddEnvVar("CODEX_PERSISTENCE", "true");
 
@@ -108,6 +125,26 @@ namespace CodexPlugin
             }
         }
 
+        private Port CreateApiPort(CodexStartupConfig config, string tag)
+        {
+            if (config.PublicTestNet == null) return AddExposedPort(tag);
+            return AddInternalPort(tag);
+        }
+
+        private Port CreateListenPort(CodexStartupConfig config)
+        {
+            if (config.PublicTestNet == null) return AddInternalPort(ListenPortTag);
+
+            return AddExposedPort(config.PublicTestNet.PublicListenPort, ListenPortTag);
+        }
+
+        private Port CreateDiscoveryPort(CodexStartupConfig config)
+        {
+            if (config.PublicTestNet == null) return AddInternalPort(DiscoveryPortTag, PortProtocol.UDP);
+
+            return AddExposedPort(config.PublicTestNet.PublicDiscoveryPort, DiscoveryPortTag, PortProtocol.UDP);
+        }
+
         private ByteSize GetVolumeCapacity(CodexStartupConfig config)
         {
             if (config.StorageQuota != null) return config.StorageQuota;

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -1,29 +1,56 @@
-using GethPlugin;
+using CodexContractsPlugin;
-using KubernetesWorkflow;
+using GethPlugin;
+using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {
     public class CodexDeployment
     {
-        public CodexDeployment(RunningContainer[] codexContainers, GethDeployment gethDeployment, RunningContainer? prometheusContainer, DeploymentMetadata metadata)
+        public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
+            CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer,
+            RunningContainers? discordBotContainer, DeploymentMetadata metadata,
+            String id)
         {
-            CodexContainers = codexContainers;
+            Id = id;
+            CodexInstances = codexInstances;
             GethDeployment = gethDeployment;
+            CodexContractsDeployment = codexContractsDeployment;
             PrometheusContainer = prometheusContainer;
+            DiscordBotContainer = discordBotContainer;
             Metadata = metadata;
         }
 
-        public RunningContainer[] CodexContainers { get; }
+        public String Id { get; }
+        public CodexInstance[] CodexInstances { get; }
         public GethDeployment GethDeployment { get; }
-        public RunningContainer? PrometheusContainer { get; }
+        public CodexContractsDeployment CodexContractsDeployment { get; }
+        public RunningContainers? PrometheusContainer { get; }
+        public RunningContainers? DiscordBotContainer { get; }
         public DeploymentMetadata Metadata { get; }
     }
 
+    public class CodexInstance
+    {
+        public CodexInstance(RunningContainers containers, CodexDebugResponse info)
+        {
+            Containers = containers;
+            Info = info;
+        }
+
+        public RunningContainers Containers { get; }
+        public CodexDebugResponse Info { get; }
+    }
+
     public class DeploymentMetadata
     {
-        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
+        public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace,
+            int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel,
+            int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI,
+            int blockMN)
         {
-            DeployDateTimeUtc = DateTime.UtcNow;
+            Name = name;
+            StartUtc = startUtc;
+            FinishedUtc = finishedUtc;
             KubeNamespace = kubeNamespace;
             NumberOfCodexNodes = numberOfCodexNodes;
             NumberOfValidators = numberOfValidators;
@@ -38,7 +65,9 @@ namespace CodexPlugin
             BlockMN = blockMN;
         }
 
-        public DateTime DeployDateTimeUtc { get; }
+        public string Name { get; }
+        public DateTime StartUtc { get; }
+        public DateTime FinishedUtc { get; }
         public string KubeNamespace { get; }
         public int NumberOfCodexNodes { get; }
         public int NumberOfValidators { get; }
@@ -52,4 +81,4 @@ namespace CodexPlugin
         public int BlockMI { get; }
         public int BlockMN { get; }
     }
-}
+}

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -2,6 +2,7 @@
 using FileUtils;
 using GethPlugin;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using MetricsPlugin;
 using Utils;
@@ -13,13 +14,18 @@ namespace CodexPlugin
         string GetName();
         CodexDebugResponse GetDebugInfo();
         CodexDebugPeerResponse GetDebugPeer(string peerId);
-        CodexDebugBlockExchangeResponse GetDebugBlockExchange();
+        // These debug methods are not available in master-line Codex. Use only for custom builds.
+        //CodexDebugBlockExchangeResponse GetDebugBlockExchange();
+        //CodexDebugRepoStoreResponse[] GetDebugRepoStore();
         ContentId UploadFile(TrackedFile file);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
+        CodexLocalData[] LocalFiles();
         void ConnectToPeer(ICodexNode node);
         CodexDebugVersionResponse Version { get; }
         IMarketplaceAccess Marketplace { get; }
         CrashWatcher CrashWatcher { get; }
+        PodInfo GetPodInfo();
+        ITransferSpeeds TransferSpeeds { get; }
         void Stop();
     }
 
@@ -29,6 +35,7 @@ namespace CodexPlugin
         private const string UploadFailedMessage = "Unable to store block";
         private readonly IPluginTools tools;
         private readonly EthAddress? ethAddress;
+        private readonly TransferSpeeds transferSpeeds;
 
         public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAddress? ethAddress)
         {
@@ -38,6 +45,7 @@ namespace CodexPlugin
             Group = group;
             Marketplace = marketplaceAccess;
             Version = new CodexDebugVersionResponse();
+            transferSpeeds = new TransferSpeeds();
         }
 
         public RunningContainer Container { get { return CodexAccess.Container; } }
@@ -46,13 +54,12 @@ namespace CodexPlugin
         public CodexNodeGroup Group { get; }
         public IMarketplaceAccess Marketplace { get; }
         public CodexDebugVersionResponse Version { get; private set; }
+        public ITransferSpeeds TransferSpeeds { get => transferSpeeds; }
         public IMetricsScrapeTarget MetricsScrapeTarget
         {
             get
             {
-                var port = CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag);
+                return new MetricsScrapeTarget(CodexAccess.Container, CodexContainerRecipe.MetricsPortTag);
-                if (port == null) throw new Exception("Metrics is not available for this Codex node. Please start it with the option '.EnableMetrics()' to enable it.");
-                return new MetricsScrapeTarget(CodexAccess.Container, port);
             }
         }
         public EthAddress EthAddress 
@@ -87,17 +94,25 @@ namespace CodexPlugin
             return CodexAccess.GetDebugBlockExchange();
         }
 
+        public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
+        {
+            return CodexAccess.GetDebugRepoStore();
+        }
+
         public ContentId UploadFile(TrackedFile file)
         {
             using var fileStream = File.OpenRead(file.Filename);
 
             var logMessage = $"Uploading file {file.Describe()}...";
             Log(logMessage);
-            var response = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
+            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
             {
                 return CodexAccess.UploadFile(fileStream);
             });
 
+            var response = measurement.Value;
+            transferSpeeds.AddUploadSample(file.GetFilesize(), measurement.Duration);
+
             if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
             if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
 
@@ -110,11 +125,17 @@ namespace CodexPlugin
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
             Log(logMessage);
             var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
-            Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file));
+            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file));
+            transferSpeeds.AddDownloadSample(file.GetFilesize(), measurement);
             Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
             return file;
         }
 
+        public CodexLocalData[] LocalFiles()
+        {
+            return CodexAccess.LocalFiles().Select(l => new CodexLocalData(new ContentId(l.cid), l.manifest)).ToArray();
+        }
+
         public void ConnectToPeer(ICodexNode node)
         {
             var peer = (CodexNode)node;
@@ -127,6 +148,11 @@ namespace CodexPlugin
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
+        public PodInfo GetPodInfo()
+        {
+            return CodexAccess.GetPodInfo();
+        }
+
         public void Stop()
         {
             if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
@@ -147,8 +173,9 @@ namespace CodexPlugin
                 throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}");
             }
 
-            //lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
+            var log = tools.GetLog();
-            //lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
+            log.AddStringReplace(nodePeerId, nodeName);
+            log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
             Version = debugInfo.codex;
         }
 
@@ -159,7 +186,10 @@ namespace CodexPlugin
 
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip);
+            var workflow = tools.CreateWorkflow();
+            var podInfo = workflow.GetPodInfo(peer.Container);
+
+            return multiAddress.Replace("0.0.0.0", podInfo.Ip);
         }
 
         private void DownloadToFile(string contentId, TrackedFile file)
@@ -191,5 +221,15 @@ namespace CodexPlugin
         }
 
         public string Id { get; }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is ContentId id && Id == id.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
     }
 }

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -1,6 +1,7 @@
 using Core;
 using GethPlugin;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using MetricsPlugin;
 using System.Collections;
 

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {
@@ -19,12 +19,13 @@ namespace CodexPlugin
 
         public void Announce()
         {
-            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}'");
+            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
         }
 
         public void AddMetadata(IAddMetadata metadata)
         {
             metadata.Add("codexid", codexStarter.GetCodexId());
+            metadata.Add("codexrevision", codexStarter.GetCodexRevision());
         }
 
         public void Decommission()
@@ -52,7 +53,7 @@ namespace CodexPlugin
             foreach (var node in result)
             {
                 mconfig.GethNode.SendEth(node, mconfig.InitialEth);
-                mconfig.CodexContracts.MintTestTokens(mconfig.GethNode, node, mconfig.InitialTokens);
+                mconfig.CodexContracts.MintTestTokens(node, mconfig.InitialTokens);
             }
         }
 

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -22,10 +22,18 @@ namespace CodexPlugin
         /// Provides an invalid proof every N proofs
         /// </summary>
         ICodexSetup WithSimulateProofFailures(uint failEveryNProofs);
+        ICodexSetup AsPublicTestNet(CodexTestNetConfig testNetConfig);
     }
 
     public class CodexLogCustomTopics
     {
+        public CodexLogCustomTopics(CodexLogLevel discV5, CodexLogLevel libp2p, CodexLogLevel blockExchange)
+        {
+            DiscV5 = discV5;
+            Libp2p = libp2p;
+            BlockExchange = blockExchange;
+        }
+
         public CodexLogCustomTopics(CodexLogLevel discV5, CodexLogLevel libp2p)
         {
             DiscV5 = discV5;
@@ -34,6 +42,7 @@ namespace CodexPlugin
 
         public CodexLogLevel DiscV5 { get; set; }
         public CodexLogLevel Libp2p { get; set; }
+        public CodexLogLevel? BlockExchange { get; }
     }
 
     public class CodexSetup : CodexStartupConfig, ICodexSetup
@@ -118,6 +127,12 @@ namespace CodexPlugin
             return this;
         }
 
+        public ICodexSetup AsPublicTestNet(CodexTestNetConfig testNetConfig)
+        {
+            PublicTestNet = testNetConfig;
+            return this;
+        }
+
         public string Describe()
         {
             var args = string.Join(',', DescribeArgs());
@@ -126,6 +141,7 @@ namespace CodexPlugin
 
         private IEnumerable<string> DescribeArgs()
         {
+            if (PublicTestNet != null) yield return $"<!>Public TestNet with listenPort: {PublicTestNet.PublicListenPort}<!>";
             yield return $"LogLevel={LogLevelWithTopics()}";
             if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
             if (StorageQuota != null) yield return $"StorageQuota={StorageQuota}";

ProjectPlugins/CodexPlugin/CodexStarter.cs

@@ -1,5 +1,6 @@
 using Core;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 
 namespace CodexPlugin
@@ -24,8 +25,12 @@ namespace CodexPlugin
 
             var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
 
-            var podInfos = string.Join(", ", containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
+            foreach (var rc in containers)
-            Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers().First().Recipe.Image}'. ({podInfos})");
+            {
+                var podInfo = GetPodInfo(rc);
+                var podInfos = string.Join(", ", rc.Containers.Select(c => $"Container: '{c.Name}' runs at '{podInfo.K8SNodeName}'={podInfo.Ip}"));
+                Log($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers().First().Recipe.Image}'. ({podInfos})");
+            }
             LogSeparator();
 
             return containers;
@@ -61,6 +66,12 @@ namespace CodexPlugin
             return recipe.Image;
         }
 
+        public string GetCodexRevision()
+        {
+            if (versionResponse != null) return versionResponse.revision;
+            return "unknown";
+        }
+
         private StartupConfig CreateStartupConfig(CodexSetup codexSetup)
         {
             var startupConfig = new StartupConfig();
@@ -80,6 +91,12 @@ namespace CodexPlugin
             return result.ToArray();
         }
 
+        private PodInfo GetPodInfo(RunningContainers rc)
+        {
+            var workflow = pluginTools.CreateWorkflow();
+            return workflow.GetPodInfo(rc);
+        }
+
         private CodexNodeGroup CreateCodexGroup(CoreInterface coreInterface, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(this, pluginTools, runningContainers, codexNodeFactory);

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -8,7 +8,7 @@ namespace CodexPlugin
         public string? NameOverride { get; set; }
         public ILocation Location { get; set; } = KnownLocations.UnspecifiedLocation;
         public CodexLogLevel LogLevel { get; set; }
-        public CodexLogCustomTopics? CustomTopics { get; set; }
+        public CodexLogCustomTopics? CustomTopics { get; set; } = new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn);
         public ByteSize? StorageQuota { get; set; }
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
@@ -18,6 +18,7 @@ namespace CodexPlugin
         public bool? EnableValidator { get; set; }
         public TimeSpan? BlockMaintenanceInterval { get; set; }
         public int? BlockMaintenanceNumber { get; set; }
+        public CodexTestNetConfig? PublicTestNet { get; set; }
 
         public string LogLevelWithTopics()
         {
@@ -51,14 +52,39 @@ namespace CodexPlugin
                     "connection",
                     "connmanager",
                     "websock",
-                    "ws-session"
+                    "ws-session",
+                    "dialer",
+                    "muxedupgrade",
+                    "upgrade",
+                    "identify"
+                };
+                var blockExchangeTopics = new[]
+                {
+                    "codex",
+                    "pendingblocks",
+                    "peerctxstore",
+                    "discoveryengine",
+                    "blockexcengine",
+                    "blockexcnetwork",
+                    "blockexcnetworkpeer"
                 };
 
                 level = $"{level};" +
                     $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" +
                     $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)}";
+
+                if (CustomTopics.BlockExchange != null)
+                {
+                    level += $";{CustomTopics.BlockExchange.ToString()!.ToLowerInvariant()}:{string.Join(",", blockExchangeTopics)}";
+                }
             }
             return level;
         }
     }
+
+    public class CodexTestNetConfig
+    {
+        public int PublicDiscoveryPort { get; set; }
+        public int PublicListenPort { get; set; }
+    }
 }

ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {
@@ -10,13 +10,6 @@ namespace CodexPlugin
             return Plugin(ci).DeployCodexNodes(number, setup);
         }
 
-        public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningContainer[] containers)
-        {
-            // ew, clean this up.
-            var rcs = new RunningContainers(null!, containers.First().Pod, containers);
-            return WrapCodexContainers(ci, new[] { rcs });
-        }
-
         public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, RunningContainers[] containers)
         {
             return Plugin(ci).WrapCodexContainers(ci, containers);

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -10,6 +10,7 @@ namespace CodexPlugin
     {
         string MakeStorageAvailable(ByteSize size, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration);
         StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
+        StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, TimeSpan expiry);
     }
 
     public class MarketplaceAccess : IMarketplaceAccess
@@ -25,13 +26,20 @@ namespace CodexPlugin
 
         public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
         {
+            return RequestStorage(contentId, pricePerSlotPerSecond, requiredCollateral, minRequiredNumberOfNodes, proofProbability, duration, duration / 2);
+        }
+
+        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, TimeSpan expiry)
+        {
+            var expireUtc = DateTimeOffset.UtcNow.ToUnixTimeSeconds() + expiry.TotalSeconds;
+
             var request = new CodexSalesRequestStorageRequest
             {
                 duration = ToDecInt(duration.TotalSeconds),
                 proofProbability = ToDecInt(proofProbability),
                 reward = ToDecInt(pricePerSlotPerSecond),
                 collateral = ToDecInt(requiredCollateral),
-                expiry = null,
+                expiry = ToDecInt(expireUtc),
                 nodes = minRequiredNumberOfNodes,
                 tolerance = null,
             };
@@ -41,11 +49,15 @@ namespace CodexPlugin
                 $"requiredCollateral: {requiredCollateral}, " +
                 $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
                 $"proofProbability: {proofProbability}, " +
+                $"expiry: {Time.FormatDuration(expiry)}, " +
                 $"duration: {Time.FormatDuration(duration)})");
 
             var response = codexAccess.RequestStorage(request, contentId.Id);
 
-            if (response == "Purchasing not available")
+            if (response == "Purchasing not available" || 
+                response == "Expiry required" ||
+                response == "Expiry needs to be in future" ||
+                response == "Expiry has to be before the request's end (now + duration)")
             {
                 throw new InvalidOperationException(response);
             }
@@ -104,6 +116,12 @@ namespace CodexPlugin
             return null!;
         }
 
+        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, TimeSpan expiry)
+        {
+            Unavailable();
+            throw new NotImplementedException();
+        }
+
         public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration)
         {
             Unavailable();
@@ -205,7 +223,7 @@ namespace CodexPlugin
 
                 if (DateTime.UtcNow - waitStart > timeout)
                 {
-                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within timeout. {statusJson}");
+                    FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within {Time.FormatDuration(timeout)} timeout. {statusJson}");
                 }
             }
             log.Log($"Contract '{desiredState}'.");