POC: codex commit blamer

This PR removes the notion of a run id and replaces it with a deploy id in continuous tests. Deploy ids can be set at deploy time only (duh), and will be picked up by the test runner from the deploy file on subsequent runs of the continuous test runner. As a consequence, the deploy id becomes a deployer parameter, and can no longer be overridden at the runner. For non-continuous tests, the deploy ID is created on-the-fly.

.github/workflows/continuous-tests.yaml

@@ -1,117 +0,0 @@
-name: Run Continuous Tests
-
-
-on:
-  # push:
-  #   branches:
-  #     - master
-  #   tags:
-  #     - 'v*.*.*'
-  #   paths-ignore:
-  #     - '**/*.md'
-  #     - '.gitignore'
-  #     - 'docker/**'
-  #     - '!docker/continuous-tests-job.yaml'
-  workflow_dispatch:
-    inputs:
-      source:
-        description: Repository with tests (current)
-        required: false
-        type: string
-      branch:
-        description: Branch with tests (master)
-        required: false
-        type: string
-      nameprefix:
-        description: Runner name prefix (c-tests-runner)
-        required: false
-        type: string
-      namespace:
-        description: Runner namespace (default)
-        required: false
-        type: string
-      tests_target_duration:
-        description: Runner target duration (172800 = 48h)
-        required: false
-        type: string
-      tests_filter:
-        description: Runner tests filter ("")
-        required: false
-        type: string
-      tests_cleanup:
-        description: Runner tests cleanup
-        type: choice
-        options:
-        - true
-        - false
-        default: true
-      deployment_namespace:
-        description: Deployment namespace (c-tests-$runid)
-        required: false
-        type: string
-
-
-env:
-  BRANCH: ${{ github.ref_name }}
-  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
-  NAMEPREFIX: c-tests-runner
-  NAMESPACE: default
-  DEPLOYMENT_NAMESPACE: c-tests
-  TESTS_TARGET_DURATION: 172800
-  TESTS_FILTER: ""
-  TESTS_CLEANUP: true
-  JOB_MANIFEST: docker/continuous-tests-job.yaml
-  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
-  KUBE_VERSION: v1.28.2
-
-
-jobs:
-  run_tests:
-    name: Run Tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Variables
-        run: |
-          RUNID=$(date +%Y%m%d-%H%M%S)
-          echo "RUNID=${RUNID}" >> $GITHUB_ENV
-          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
-          [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ github.event.inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ github.event.inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ github.event.inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.deployment_namespace }}" ]] && echo "DEPLOYMENT_NAMESPACE=${{ github.event.inputs.deployment_namespace }}" >>"$GITHUB_ENV" || echo "DEPLOYMENT_NAMESPACE=${{ env.DEPLOYMENT_NAMESPACE }}-${RUNID}" >>"$GITHUB_ENV"
-
-      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
-        uses: azure/setup-kubectl@v3
-        with:
-          version: ${{ env.KUBE_VERSION }}
-
-      - name: Kubectl - Kubeconfig
-        run: |
-          mkdir -p "${HOME}"/.kube
-          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
-
-      - name: Kubectl - Create Job
-        run: |
-          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
-
-      - name: Tests Identification
-        run: |
-          echo "----"
-          echo "Repository: ${{ env.SOURCE }}"
-          echo "Branch: ${{ env.BRANCH }}"
-          echo "Runner Pod: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
-          echo "Runner namespace: ${{ env.NAMESPACE }}"
-          echo "----"
-          echo "Tests runid: ${{ env.RUNID }}"
-          echo "Tests namespace: ${{ env.DEPLOYMENT_NAMESPACE }}"
-          echo "Tests duration: ${{ env.TESTS_TARGET_DURATION }}"
-          echo "Tests filter: ${{ env.TESTS_FILTER }}"
-          echo "Tests cleanup: ${{ env.TESTS_CLEANUP }}"
-          echo "----"

.github/workflows/docker-discordbot.yml

@@ -0,0 +1,28 @@
+name: Docker - Discord Bot
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/BiblioTech/**'
+      - '!Tools/BiblioTech/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-discordbot.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/BiblioTech/docker/Dockerfile
+      docker_repo: codexstorage/codex-discordbot
+    secrets: inherit
+

.github/workflows/docker-reusable.yml

@@ -1,4 +1,4 @@
-name: Docker - Reusable
+name: Reusable - Docker
 
 
 on:
@@ -66,26 +66,26 @@ jobs:
       PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Docker - Meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.DOCKER_REPO }}
 
       - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Docker - Build and Push by digest
         id: build
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           file: ${{ env.DOCKER_FILE }}
@@ -101,9 +101,9 @@ jobs:
           touch "/tmp/digests/${digest#sha256:}"
 
       - name: Docker - Upload digest
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: digests
+          name: digests-${{ matrix.target.arch }}
           path: /tmp/digests/*
           if-no-files-found: error
           retention-days: 1
@@ -139,17 +139,18 @@ jobs:
           fi
 
       - name: Docker - Download digests
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: digests
+          pattern: digests-*
+          merge-multiple: true
           path: /tmp/digests
 
       - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Docker - Meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.DOCKER_REPO }}
           flavor: |
@@ -161,7 +162,7 @@ jobs:
             type=sha,enable=${{ env.TAG_SHA }}
 
       - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/docker-rewarder.yml

@@ -0,0 +1,26 @@
+name: Docker - Rewarder Bot
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - 'Tools/TestNetRewarder/**'
+      - '!Tools/TestNetRewarder/docker/docker-compose.yaml'
+      - 'Framework/**'
+      - 'ProjectPlugins/**'
+      - .github/workflows/docker-rewarder.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    with:
+      docker_file: Tools/TestNetRewarder/docker/Dockerfile
+      docker_repo: codexstorage/codex-rewarderbot
+    secrets: inherit
+

.github/workflows/docker-runner.yml

@@ -1,4 +1,4 @@
-name: Docker
+name: Docker - Runner
 
 
 on:
@@ -10,7 +10,7 @@ on:
     paths:
       - docker/Dockerfile
       - docker/docker-entrypoint.sh
-      - .github/workflows/docker.yml
+      - .github/workflows/docker-runner.yml
       - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 

.github/workflows/report-HoldMyBeerTest.yaml

@@ -0,0 +1,20 @@
+name: Report - HoldMyBeerTest
+
+
+on:
+  # schedule:
+  #   - cron: '30 1 */2 * *'
+  workflow_dispatch:
+
+jobs:
+  run_tests:
+    name: Call runner
+    uses: ./.github/workflows/run-continuous-tests.yaml
+    with:
+      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+      branch: master
+      nameprefix: c-tests-report-holdmybeertest-48h
+      tests_target_duration: 48h
+      tests_filter: HoldMyBeerTest
+      tests_cleanup: true
+    secrets: inherit

.github/workflows/report-PeersTest.yaml

@@ -0,0 +1,20 @@
+name: Report - PeersTest
+
+
+on:
+  # schedule:
+  #   - cron: '30 1 */2 * *'
+  workflow_dispatch:
+
+jobs:
+  run_tests:
+    name: Call runner
+    uses: ./.github/workflows/run-continuous-tests.yaml
+    with:
+      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+      branch: master
+      nameprefix: c-tests-report-peerstest-48h
+      tests_target_duration: 48h
+      tests_filter: PeersTest
+      tests_cleanup: true
+    secrets: inherit

.github/workflows/run-continuous-tests.yaml

@@ -0,0 +1,170 @@
+name: Run Continuous Tests
+
+
+on:
+  workflow_dispatch:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
+        required: false
+        type: string
+      nameprefix:
+        description: Resources prefix (c-tests)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (2d)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup
+        required: false
+        type: boolean
+        default: true
+  workflow_call:
+    inputs:
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
+      codexdockerimage:
+        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
+        required: false
+        type: string
+      nameprefix:
+        description: Resources prefix (c-tests)
+        required: false
+        type: string
+      tests_target_duration:
+        description: Runner target duration (2d)
+        required: false
+        type: string
+      tests_filter:
+        description: Runner tests filter ("")
+        required: false
+        type: string
+      tests_cleanup:
+        description: Runner tests cleanup
+        required: false
+        type: boolean
+        default: true
+      workflow_source:
+        description: Workflow source
+        required: false
+        type: string
+        default: ''
+
+
+env:
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
+  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
+  NAMEPREFIX: c-tests
+  NAMESPACE: default
+  TESTS_TARGET_DURATION: 2d
+  TESTS_FILTER: ""
+  TESTS_CLEANUP: true
+  JOB_MANIFEST: docker/continuous-tests-job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.28.2
+
+
+jobs:
+  run_tests:
+    name: Run Continuous Tests ${{ inputs.tests_filter }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.workflow_source }}
+
+      - name: Variables
+        run: |
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
+          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          echo "DEPLOYMENT_NAMESPACE=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@releases/v4.0.0
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}"
+          echo "Runner pod: `kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}'`"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+          echo "Tests runid: ${{ env.RUNID }}"
+          echo "Tests codexdockerimage: ${{ env.CODEXDOCKERIMAGE }}"
+          echo "Tests namespace: ${{ env.DEPLOYMENT_NAMESPACE }}"
+          echo "Tests duration: ${{ env.TESTS_TARGET_DURATION }}"
+          echo "Tests filter: ${{ env.TESTS_FILTER }}"
+          echo "Tests cleanup: ${{ env.TESTS_CLEANUP }}"
+          echo "----"
+
+      - name: Show Runner logs
+        if: false
+        run: |
+          # Variables
+          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
+          duration=600
+          namespace="${{ env.NAMESPACE }}"
+          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")
+
+          # Check Pod status
+          WAIT=120
+          SECONDS=0
+          sleep=1
+          while (( SECONDS < WAIT )); do
+            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath="{.status.phase}")
+            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in $phase state - Get the logs"; break; } || { echo "Pod $pod is in $phase state - Retry in $sleep seconds / $((WAIT - SECONDS))"; }
+            sleep $sleep
+          done
+
+          # Get logs
+          timeout $duration \
+            kubectl logs $pod \
+              -n $namespace \
+              -f \
+              --tail=-1 \
+              --timestamps || true

.github/workflows/run-dist-tests.yaml

@@ -2,16 +2,6 @@ name: Run Dist Tests
 
 
 on:
-  # push:
-  #   branches:
-  #     - master
-  #   tags:
-  #     - 'v*.*.*'
-  #   paths-ignore:
-  #     - '**/*.md'
-  #     - '.gitignore'
-  #     - 'docker/**'
-  #     - '!docker/dist-tests-job.yaml'
   workflow_dispatch:
     inputs:
       source:
@@ -37,8 +27,8 @@ on:
 
 
 env:
-  BRANCH: ${{ github.ref_name }}
   SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  BRANCH: ${{ github.ref_name }}
   NAMEPREFIX: d-tests-runner
   NAMESPACE: default
   COMMAND: dotnet test Tests/CodexTests
@@ -53,21 +43,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Variables
         run: |
-          [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.command }}" ]] && COMMAND="${{ github.event.inputs.command }}" || COMMAND="${{ env.COMMAND }}"
           echo "COMMAND=$(jq -c 'split(" ")' <<< '"'$COMMAND'"')" >>"$GITHUB_ENV"
           echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
           echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
 
       - name: Kubectl - Install ${{ env.KUBE_VERSION }}
-        uses: azure/setup-kubectl@v3
+        uses: azure/setup-kubectl@releases/v4.0.0
         with:
           version: ${{ env.KUBE_VERSION }}
 
@@ -85,7 +75,7 @@ jobs:
           echo "----"
           echo "Repository: ${{ env.SOURCE }}"
           echo "Branch: ${{ env.BRANCH }}"
-          echo "Runner Pod: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
+          echo "Runner job: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
           echo "Runner namespace: ${{ env.NAMESPACE }}"
           echo "----"
           echo "Tests runid: ${{ env.RUNID }}"

.gitignore

@@ -1,4 +1,5 @@
 .vs
 obj
 bin
-.vscode
+.vscode
+Tools/CommitBlamer/CodexTestLogs

Framework/DiscordRewards/CheckConfig.cs

@@ -0,0 +1,21 @@
+using Utils;
+
+namespace DiscordRewards
+{
+    public class CheckConfig
+    {
+        public CheckType Type { get; set; }
+        public ulong MinNumberOfHosts { get; set; }
+        public ByteSize MinSlotSize { get; set; } = 0.Bytes();
+        public TimeSpan MinDuration { get; set; } = TimeSpan.Zero;
+    }
+
+    public enum CheckType
+    {
+        Uninitialized,
+        FilledSlot,
+        FinishedSlot,
+        PostedContract,
+        StartedContract,
+    }
+}

Framework/DiscordRewards/DiscordRewards.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/DiscordRewards/GiveRewardsCommand.cs

@@ -0,0 +1,13 @@
+namespace DiscordRewards
+{
+    public class GiveRewardsCommand
+    {
+        public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
+    }
+
+    public class RewardUsersCommand
+    {
+        public ulong RewardId { get; set; }
+        public string[] UserAddresses { get; set; } = Array.Empty<string>();
+    }
+}

Framework/DiscordRewards/RewardConfig.cs

@@ -0,0 +1,18 @@
+namespace DiscordRewards
+{
+    public class RewardConfig
+    {
+        public const string UsernameTag = "<USER>";
+
+        public RewardConfig(ulong roleId, string message, CheckConfig checkConfig)
+        {
+            RoleId = roleId;
+            Message = message;
+            CheckConfig = checkConfig;
+        }
+
+        public ulong RoleId { get; }
+        public string Message { get; }
+        public CheckConfig CheckConfig { get; }
+    }
+}

Framework/DiscordRewards/RewardRepo.cs

@@ -0,0 +1,53 @@
+using Utils;
+
+namespace DiscordRewards
+{
+    public class RewardRepo
+    {
+        private static string Tag => RewardConfig.UsernameTag;
+
+        public RewardConfig[] Rewards { get; } = new RewardConfig[]
+        {
+            // Filled any slot
+            new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig
+            {
+                Type = CheckType.FilledSlot
+            }),
+
+            // Finished any slot
+            new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig
+            {
+                Type = CheckType.FinishedSlot
+            }),
+
+            // Finished a sizable slot
+            new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot!", new CheckConfig
+            {
+                Type = CheckType.FinishedSlot,
+                MinSlotSize = 1.GB(),
+                MinDuration = TimeSpan.FromHours(24.0),
+            }),
+
+            // Posted any contract
+            new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig
+            {
+                Type = CheckType.PostedContract
+            }),
+
+            // Started any contract
+            new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig
+            {
+                Type = CheckType.StartedContract
+            }),
+
+            // Started a sizable contract
+            new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time!", new CheckConfig
+            {
+                Type = CheckType.FinishedSlot,
+                MinNumberOfHosts = 4,
+                MinSlotSize = 1.GB(),
+                MinDuration = TimeSpan.FromHours(24.0),
+            })
+        };
+    }
+}

Framework/FileUtils/TrackedFile.cs

@@ -37,6 +37,11 @@ namespace FileUtils
             return $"'{Filename}'{sizePostfix}";
         }
 
+        public ByteSize GetFilesize()
+        {
+            return new ByteSize(GetFileSize());
+        }
+
         private void AssertEqual(TrackedFile? actual)
         {
             if (actual == null)  FrameworkAssert.Fail("TestFile is null.");

Framework/GethConnector/GethConnector.cs

@@ -0,0 +1,39 @@
+using CodexContractsPlugin;
+using GethPlugin;
+using Logging;
+
+namespace GethConnector
+{
+    public class GethConnector
+    {
+        public IGethNode GethNode { get; }
+        public ICodexContracts CodexContracts { get; }
+
+        public static GethConnector? Initialize(ILog log)
+        {
+            if (!string.IsNullOrEmpty(GethInput.LoadError))
+            {
+                var msg = "Geth input incorrect: " + GethInput.LoadError;
+                log.Error(msg);
+                return null;
+            }
+
+            var contractsDeployment = new CodexContractsDeployment(
+                marketplaceAddress: GethInput.MarketplaceAddress,
+                abi: GethInput.ABI,
+                tokenAddress: GethInput.TokenAddress
+            );
+
+            var gethNode = new CustomGethNode(log, GethInput.GethHost, GethInput.GethPort, GethInput.PrivateKey);
+            var contracts = new CodexContractsAccess(log, gethNode, contractsDeployment);
+
+            return new GethConnector(gethNode, contracts);
+        }
+
+        private GethConnector(IGethNode gethNode, ICodexContracts codexContracts)
+        {
+            GethNode = gethNode;
+            CodexContracts = codexContracts;
+        }
+    }
+}

Framework/GethConnector/GethConnector.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Logging\Logging.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\CodexContractsPlugin\CodexContractsPlugin.csproj" />
+    <ProjectReference Include="..\..\ProjectPlugins\GethPlugin\GethPlugin.csproj" />
+  </ItemGroup>
+
+</Project>

Framework/GethConnector/GethInput.cs

@@ -0,0 +1,52 @@
+namespace GethConnector
+{
+    public static class GethInput
+    {
+        private const string GethHostVar = "GETH_HOST";
+        private const string GethPortVar = "GETH_HTTP_PORT";
+        private const string GethPrivKeyVar = "GETH_PRIVATE_KEY";
+        private const string MarketplaceAddressVar = "CODEXCONTRACTS_MARKETPLACEADDRESS";
+        private const string TokenAddressVar = "CODEXCONTRACTS_TOKENADDRESS";
+        private const string AbiVar = "CODEXCONTRACTS_ABI";
+
+        static GethInput()
+        {
+            var error = new List<string>();
+            var gethHost = GetEnvVar(error, GethHostVar);
+            var gethPort = Convert.ToInt32(GetEnvVar(error, GethPortVar));
+            var privateKey = GetEnvVar(error, GethPrivKeyVar);
+            var marketplaceAddress = GetEnvVar(error, MarketplaceAddressVar);
+            var tokenAddress = GetEnvVar(error, TokenAddressVar);
+            var abi = GetEnvVar(error, AbiVar);
+
+            if (error.Any())
+            {
+                LoadError = string.Join(", ", error);
+            }
+            else
+            {
+                GethHost = gethHost!;
+                GethPort = gethPort;
+                PrivateKey = privateKey!;
+                MarketplaceAddress = marketplaceAddress!;
+                TokenAddress = tokenAddress!;
+                ABI = abi!;
+            }
+        }
+
+        public static string GethHost { get; } = string.Empty;
+        public static int GethPort { get; }
+        public static string PrivateKey { get; } = string.Empty;
+        public static string MarketplaceAddress { get; } = string.Empty;
+        public static string TokenAddress { get; } = string.Empty;
+        public static string ABI { get; } = string.Empty;
+        public static string LoadError { get; } = string.Empty;
+
+        private static string? GetEnvVar(List<string> error, string name)
+        {
+            var result = Environment.GetEnvironmentVariable(name);
+            if (string.IsNullOrEmpty(result)) error.Add($"'{name}' is not set.");
+            return result;
+        }
+    }
+}

Framework/KubernetesWorkflow/K8sController.cs

@@ -84,6 +84,31 @@ namespace KubernetesWorkflow
             return result;
         }
 
+        public int[] GetUsedExternalPorts()
+        {
+            return client.Run(c =>
+            {
+                var result = new List<int>();
+
+                var services = c.ListServiceForAllNamespaces();
+                var nodePorts = services.Items.Where(s => s.Spec.Type == "NodePort").ToArray();
+                if (!nodePorts.Any()) return result.ToArray();
+
+                foreach (var service in nodePorts)
+                {
+                    foreach (var port in service.Spec.Ports)
+                    {
+                        if (port.NodePort.HasValue)
+                        {
+                            result.Add(port.NodePort.Value);
+                        }
+                    }
+                }
+
+                return result.ToArray();
+            });
+        }
+
         public void DeleteAllNamespacesStartingWith(string prefix)
         {
             log.Debug();
@@ -335,6 +360,8 @@ namespace KubernetesWorkflow
                         },
                         Spec = new V1PodSpec
                         {
+                            PriorityClassName = GetPriorityClassName(containerRecipes),
+                            Affinity = CreatePodAffinity(containerRecipes),
                             NodeSelector = CreateNodeSelector(location),
                             Containers = CreateDeploymentContainers(containerRecipes),
                             Volumes = CreateVolumes(containerRecipes)
@@ -367,12 +394,57 @@ namespace KubernetesWorkflow
             };
         }
 
+        private V1Affinity? CreatePodAffinity(ContainerRecipe[] recipes)
+        {
+            var notIns = recipes
+                .Select(r => r.SchedulingAffinity.NotIn)
+                .Where(n => !string.IsNullOrEmpty(n))
+                .Distinct()
+                .ToList();
+
+            if (!notIns.Any()) return null;
+
+            return new V1Affinity
+            {
+                NodeAffinity = new V1NodeAffinity
+                {
+                    RequiredDuringSchedulingIgnoredDuringExecution = new V1NodeSelector
+                    {
+                        NodeSelectorTerms = new List<V1NodeSelectorTerm>
+                        { 
+                            new V1NodeSelectorTerm
+                            {
+                                MatchExpressions = new List<V1NodeSelectorRequirement>
+                                {
+                                    new V1NodeSelectorRequirement
+                                    {
+                                        Key = "allow-tests-pods",
+                                        OperatorProperty = "NotIn",
+                                        Values = notIns
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            };
+        }
+
         private K8sNodeLabel? GetNodeLabelForLocation(ILocation location)
         {
             var l = (Location)location;
             return l.NodeLabel;
         }
 
+        private string GetPriorityClassName(ContainerRecipe[] containerRecipes)
+        {
+            if (containerRecipes.Any(c => c.SetCriticalPriority))
+            {
+                return "system-node-critical";
+            }
+            return null!;
+        }
+
         private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes)
         {
             return containerRecipes.First().PodLabels.GetLabels();
@@ -627,7 +699,8 @@ namespace KubernetesWorkflow
         private V1Pod GetPodForDeployment(RunningDeployment deployment)
         {
             return Time.Retry(() => GetPodForDeplomentInternal(deployment),
-                maxRetries: 2,
+                // We will wait up to 1 minute, k8s might be moving pods around.
+                maxRetries: 6,
                 retryTime: TimeSpan.FromSeconds(10),
                 description: "Find pod by label for deployment.");
         }
@@ -656,17 +729,17 @@ namespace KubernetesWorkflow
 
         private RunningService? CreateInternalService(ContainerRecipe[] recipes)
         {
-            return CreateService(recipes, r => r.InternalPorts.Concat(r.ExposedPorts).ToArray(), "ClusterIP", "int");
+            return CreateService(recipes, r => r.InternalPorts.Concat(r.ExposedPorts).ToArray(), "ClusterIP", "int", false);
         }
 
         private RunningService? CreateExternalService(ContainerRecipe[] recipes)
         {
-            return CreateService(recipes, r => r.ExposedPorts, "NodePort", "ext");
+            return CreateService(recipes, r => r.ExposedPorts, "NodePort", "ext", true);
         }
 
-        private RunningService? CreateService(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector, string serviceType, string namePostfix)
+        private RunningService? CreateService(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector, string serviceType, string namePostfix, bool isNodePort)
         {
-            var ports = CreateServicePorts(recipes, portSelector);
+            var ports = CreateServicePorts(recipes, portSelector, isNodePort);
             if (!ports.Any()) return null;
 
             var serviceSpec = new V1Service
@@ -740,7 +813,7 @@ namespace KubernetesWorkflow
             };
         }
 
-        private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector)
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe[] recipes, Func<ContainerRecipe, Port[]> portSelector, bool isNodePort)
         {
             var result = new List<V1ServicePort>();
             foreach (var recipe in recipes)
@@ -748,29 +821,33 @@ namespace KubernetesWorkflow
                 var ports = portSelector(recipe);
                 foreach (var port in ports)
                 {
-                    result.AddRange(CreateServicePorts(recipe, port));
+                    result.AddRange(CreateServicePorts(recipe, port, isNodePort));
                 }
             }
             return result;
         }
 
-        private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe, Port recipePort)
+        private List<V1ServicePort> CreateServicePorts(ContainerRecipe recipe, Port recipePort, bool isNodePort)
         {
             var result = new List<V1ServicePort>();
-            if (recipePort.IsTcp()) CreateServicePort(result, recipe, recipePort, "TCP");
-            if (recipePort.IsUdp()) CreateServicePort(result, recipe, recipePort, "UDP");
+            if (recipePort.IsTcp()) CreateServicePort(result, recipe, recipePort, "TCP", isNodePort);
+            if (recipePort.IsUdp()) CreateServicePort(result, recipe, recipePort, "UDP", isNodePort);
             return result;
         }
 
-        private void CreateServicePort(List<V1ServicePort> result, ContainerRecipe recipe, Port port, string protocol)
+        private void CreateServicePort(List<V1ServicePort> result, ContainerRecipe recipe, Port port, string protocol, bool isNodePort)
         {
-            result.Add(new V1ServicePort
+            var p = new V1ServicePort
             {
                 Name = GetNameForPort(recipe, port),
                 Protocol = protocol,
                 Port = port.Number,
                 TargetPort = GetNameForPort(recipe, port)
-            });
+            };
+
+            if (isNodePort) p.NodePort = port.Number;
+
+            result.Add(p);
         }
 
         #endregion

Framework/KubernetesWorkflow/PublicIpService.cs

@@ -0,0 +1,7 @@
+namespace KubernetesWorkflow
+{
+    public static class PublicIpService
+    {
+        public static string Address { get; } = "ip.codex.storage";
+    }
+}

Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs

@@ -2,12 +2,14 @@
 {
     public class ContainerRecipe
     {
-        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
+        public ContainerRecipe(int number, string? nameOverride, string image, ContainerResources resources, SchedulingAffinity schedulingAffinity, bool setCriticalPriority, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, VolumeMount[] volumes, ContainerAdditionals additionals)
         {
             Number = number;
             NameOverride = nameOverride;
             Image = image;
             Resources = resources;
+            SchedulingAffinity = schedulingAffinity;
+            SetCriticalPriority = setCriticalPriority;
             ExposedPorts = exposedPorts;
             InternalPorts = internalPorts;
             EnvVars = envVars;
@@ -32,6 +34,8 @@
         public int Number { get; }
         public string? NameOverride { get; }
         public ContainerResources Resources { get; }
+        public SchedulingAffinity SchedulingAffinity { get; }
+        public bool SetCriticalPriority { get; }
         public string Image { get; }
         public Port[] ExposedPorts { get; }
         public Port[] InternalPorts { get; }
@@ -53,6 +57,7 @@
                 $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
                 $"envVars: {string.Join(",", EnvVars.Select(v => v.ToString()))}, " +
                 $"limits: {Resources}, " +
+                $"affinity: {SchedulingAffinity}, " +
                 $"volumes: {string.Join(",", Volumes.Select(v => $"'{v.MountPath}'"))}";
         }
     }

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -13,6 +13,8 @@ namespace KubernetesWorkflow.Recipe
         private readonly List<object> additionals = new List<object>();
         private RecipeComponentFactory factory = null!;
         private ContainerResources resources = new ContainerResources();
+        private SchedulingAffinity schedulingAffinity = new SchedulingAffinity();
+        private bool setCriticalPriority;
 
         public ContainerRecipe CreateRecipe(int index, int containerNumber, RecipeComponentFactory factory, StartupConfig config)
         {
@@ -22,7 +24,7 @@ namespace KubernetesWorkflow.Recipe
 
             Initialize(config);
 
-            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources,
+            var recipe = new ContainerRecipe(containerNumber, config.NameOverride, Image, resources, schedulingAffinity, setCriticalPriority,
                 exposedPorts.ToArray(),
                 internalPorts.ToArray(),
                 envVars.ToArray(),
@@ -40,6 +42,8 @@ namespace KubernetesWorkflow.Recipe
             additionals.Clear();
             this.factory = null!;
             resources = new ContainerResources();
+            schedulingAffinity = new SchedulingAffinity();
+            setCriticalPriority = false;
 
             return recipe;
         }
@@ -52,17 +56,17 @@ namespace KubernetesWorkflow.Recipe
 
         protected Port AddExposedPort(string tag, PortProtocol protocol = PortProtocol.TCP)
         {
-            return AddExposedPort(factory.CreatePort(tag, protocol));
+            return AddExposedPort(factory.CreateExternalPort(tag, protocol));
         }
 
         protected Port AddExposedPort(int number, string tag, PortProtocol protocol = PortProtocol.TCP)
         {
-            return AddExposedPort(factory.CreatePort(number, tag, protocol));
+            return AddExposedPort(factory.CreateExternalPort(number, tag, protocol));
         }
 
         protected Port AddInternalPort(string tag = "", PortProtocol protocol = PortProtocol.TCP)
         {
-            var p = factory.CreatePort(tag, protocol);
+            var p = factory.CreateInternalPort(tag, protocol);
             internalPorts.Add(p);
             return p;
         }
@@ -121,6 +125,16 @@ namespace KubernetesWorkflow.Recipe
             SetResourcesRequest(new ContainerResourceSet(milliCPUs, memory));
         }
 
+        protected void SetSchedulingAffinity(string notIn)
+        {
+            schedulingAffinity = new SchedulingAffinity(notIn);
+        }
+
+        protected void SetSystemCriticalPriority()
+        {
+            setCriticalPriority = true;
+        }
+
         // Disabled following a possible bug in the k8s cluster that will throttle containers much more than is
         // called for if they have resource limits defined.
         //protected void SetResourceLimits(int milliCPUs, ByteSize memory)

Framework/KubernetesWorkflow/Recipe/RecipeComponentFactory.cs

@@ -5,16 +5,36 @@ namespace KubernetesWorkflow.Recipe
 {
     public class RecipeComponentFactory
     {
-        private NumberSource portNumberSource = new NumberSource(8080);
+        private readonly NumberSource internalNumberSource = new NumberSource(8080);
+        private static readonly NumberSource externalNumberSource = new NumberSource(30000);
+        private static int[] usedExternalPorts = Array.Empty<int>();
 
-        public Port CreatePort(int number, string tag, PortProtocol protocol)
+        public void Update(K8sController controller)
         {
+            usedExternalPorts = controller.GetUsedExternalPorts();
+        }
+
+        public Port CreateInternalPort(string tag, PortProtocol protocol)
+        {
+            return new Port(internalNumberSource.GetNextNumber(), tag, protocol);
+        }
+
+        public Port CreateExternalPort(int number, string tag, PortProtocol protocol)
+        {
+            if (usedExternalPorts.Contains(number)) throw new Exception($"External port number {number} is already in use by the cluster.");
             return new Port(number, tag, protocol);
         }
 
-        public Port CreatePort(string tag, PortProtocol protocol)
+        public Port CreateExternalPort(string tag, PortProtocol protocol)
         {
-            return new Port(portNumberSource.GetNextNumber(), tag, protocol);
+            while (true)
+            {
+                var number = externalNumberSource.GetNextNumber();
+                if (!usedExternalPorts.Contains(number))
+                {
+                    return new Port(number, tag, protocol);
+                }
+            }
         }
 
         public EnvVar CreateEnvVar(string name, int value)

Framework/KubernetesWorkflow/Recipe/SchedulingAffinity.cs

@@ -0,0 +1,18 @@
+namespace KubernetesWorkflow.Recipe
+{
+    public class SchedulingAffinity
+    {
+        public SchedulingAffinity(string? notIn = null)
+        {
+            NotIn = notIn;
+        }
+
+        public string? NotIn { get; }
+
+        public override string ToString()
+        {
+            if (string.IsNullOrEmpty(NotIn)) return "none";
+            return "notIn:" + NotIn;
+        }
+    }
+}

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -54,12 +54,19 @@ namespace KubernetesWorkflow
         {
             return K8s(controller =>
             {
+                componentFactory.Update(controller);
+
                 var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
                 var startResult = controller.BringOnline(recipes, location);
                 var containers = CreateContainers(startResult, recipes, startupConfig);
 
                 var rc = new RunningContainers(startupConfig, startResult, containers);
                 cluster.Configuration.Hooks.OnContainersStarted(rc);
+
+                if (startResult.ExternalService != null)
+                {
+                    componentFactory.Update(controller);
+                }
                 return rc;
             });
         }
@@ -222,7 +229,7 @@ namespace KubernetesWorkflow
             }
             catch (k8s.Autorest.HttpOperationException ex)
             {
-                log.Error(JsonConvert.SerializeObject(ex));
+                log.Error(JsonConvert.SerializeObject(ex.Response));
                 throw;
             }
         }
@@ -238,7 +245,7 @@ namespace KubernetesWorkflow
             }
             catch (k8s.Autorest.HttpOperationException ex)
             {
-                log.Error(JsonConvert.SerializeObject(ex));
+                log.Error(JsonConvert.SerializeObject(ex.Response));
                 throw;
             }
         }

Framework/Logging/BaseLog.cs

@@ -55,7 +55,9 @@ namespace Logging
 
         public virtual void Error(string message)
         {
-            Log($"[ERROR] {message}");
+            var msg = $"[ERROR] {message}";
+            Console.WriteLine(msg);
+            Log(msg);
         }
 
         public virtual void AddStringReplace(string from, string to)

Framework/Logging/FileLog.cs

@@ -0,0 +1,17 @@
+namespace Logging
+{
+    public class FileLog : BaseLog
+    {
+        public FileLog(string fullFilename)
+        {
+            FullFilename = fullFilename;
+        }
+
+        public string FullFilename { get; }
+
+        protected override string GetFullName()
+        {
+            return FullFilename;
+        }
+    }
+}

Framework/Logging/NullLog.cs

@@ -16,6 +16,7 @@
 
         public override void Error(string message)
         {
+            Console.WriteLine("Error: " + message);
             base.Error(message);
         }
 

Framework/Logging/Stopwatch.cs

@@ -16,19 +16,19 @@ namespace Logging
             this.debug = debug;
         }
 
-        public static void Measure(ILog log, string name, Action action, bool debug = false)
+        public static TimeSpan Measure(ILog log, string name, Action action, bool debug = false)
         {
             var sw = Begin(log, name, debug);
             action();
-            sw.End();
+            return sw.End();
         }
 
-        public static T Measure<T>(ILog log, string name, Func<T> action, bool debug = false)
+        public static StopwatchResult<T> Measure<T>(ILog log, string name, Func<T> action, bool debug = false)
         {
             var sw = Begin(log, name, debug);
             var result = action();
-            sw.End();
-            return result;
+            var duration = sw.End();
+            return new StopwatchResult<T>(result, duration);
         }
 
         public static Stopwatch Begin(ILog log)
@@ -68,4 +68,16 @@ namespace Logging
             return duration;
         }
     }
+
+    public class StopwatchResult<T>
+    {
+        public StopwatchResult(T value, TimeSpan duration)
+        {
+            Value = value;
+            Duration = duration;
+        }
+
+        public T Value { get; }
+        public TimeSpan Duration { get; }
+    }
 }

Framework/NethereumWorkflow/BlockTimeEntry.cs

@@ -0,0 +1,22 @@
+namespace NethereumWorkflow
+{
+    public partial class BlockTimeFinder
+    {
+        public class BlockTimeEntry
+        {
+            public BlockTimeEntry(ulong blockNumber, DateTime utc)
+            {
+                BlockNumber = blockNumber;
+                Utc = utc;
+            }
+
+            public ulong BlockNumber { get; }
+            public DateTime Utc { get; }
+
+            public override string ToString()
+            {
+                return $"[{BlockNumber}] @ {Utc.ToString("o")}";
+            }
+        }
+    }
+}

Framework/NethereumWorkflow/BlockTimeFinder.cs

@@ -0,0 +1,280 @@
+using Logging;
+using Nethereum.RPC.Eth.DTOs;
+using Nethereum.Web3;
+using Utils;
+
+namespace NethereumWorkflow
+{
+    public partial class BlockTimeFinder
+    {
+        private const ulong FetchRange = 6;
+        private const int MaxEntries = 1024;
+        private static readonly Dictionary<ulong, BlockTimeEntry> entries = new Dictionary<ulong, BlockTimeEntry>();
+        private readonly Web3 web3;
+        private readonly ILog log;
+        
+        public BlockTimeFinder(Web3 web3, ILog log)
+        {
+            this.web3 = web3;
+            this.log = log;
+        }
+
+        public ulong GetHighestBlockNumberBefore(DateTime moment)
+        {
+            log.Log("Looking for highest block before " + moment.ToString("o"));
+            AssertMomentIsInPast(moment);
+            Initialize();
+
+            return GetHighestBlockBefore(moment);
+        }
+
+        public ulong GetLowestBlockNumberAfter(DateTime moment)
+        {
+            log.Log("Looking for lowest block after " + moment.ToString("o"));
+            AssertMomentIsInPast(moment);
+            Initialize();
+
+            return GetLowestBlockAfter(moment);
+        }
+
+        private ulong GetHighestBlockBefore(DateTime moment)
+        {
+            var closestBefore = FindClosestBeforeEntry(moment);
+            var closestAfter = FindClosestAfterEntry(moment);
+
+            if (closestBefore != null &&
+                closestAfter != null &&
+                closestBefore.Utc < moment &&
+                closestAfter.Utc > moment &&
+                closestBefore.BlockNumber + 1 == closestAfter.BlockNumber)
+            {
+                log.Log("Found highest-Before: " + closestBefore);
+                return closestBefore.BlockNumber;
+            }
+
+            FetchBlocksAround(moment);
+            return GetHighestBlockBefore(moment);
+        }
+
+        private ulong GetLowestBlockAfter(DateTime moment)
+        {
+            var closestBefore = FindClosestBeforeEntry(moment);
+            var closestAfter = FindClosestAfterEntry(moment);
+
+            if (closestBefore != null &&
+                closestAfter != null &&
+                closestBefore.Utc < moment &&
+                closestAfter.Utc > moment &&
+                closestBefore.BlockNumber + 1 == closestAfter.BlockNumber)
+            {
+                log.Log("Found lowest-after: " + closestAfter);
+                return closestAfter.BlockNumber;
+            }
+
+            FetchBlocksAround(moment);
+            return GetLowestBlockAfter(moment);
+        }
+
+        private void FetchBlocksAround(DateTime moment)
+        {
+            var timePerBlock = EstimateTimePerBlock();
+            log.Debug("Fetching blocks around " + moment.ToString("o") + " timePerBlock: " + timePerBlock.TotalSeconds);
+
+            EnsureRecentBlockIfNecessary(moment, timePerBlock);
+
+            var max = entries.Keys.Max();
+            var blockDifference = CalculateBlockDifference(moment, timePerBlock, max);
+
+            FetchUp(max, blockDifference);
+            FetchDown(max, blockDifference);
+        }
+
+        private void FetchDown(ulong max, ulong blockDifference)
+        {
+            var target = max - blockDifference - 1;
+            var fetchDown = FetchRange;
+            while (fetchDown > 0)
+            {
+                if (!entries.ContainsKey(target))
+                {
+                    var newBlock = AddBlockNumber(target);
+                    if (newBlock == null) return;
+                    fetchDown--;
+                }
+                target--;
+                if (target <= 0) return;
+            }
+        }
+
+        private void FetchUp(ulong max, ulong blockDifference)
+        {
+            var target = max - blockDifference;
+            var fetchUp = FetchRange;
+            while (fetchUp > 0)
+            {
+                if (!entries.ContainsKey(target))
+                {
+                    var newBlock = AddBlockNumber(target);
+                    if (newBlock == null) return;
+                    fetchUp--;
+                }
+                target++;
+                if (target >= max) return;
+            }
+        }
+
+        private ulong CalculateBlockDifference(DateTime moment, TimeSpan timePerBlock, ulong max)
+        {
+            var latest = entries[max];
+            var timeDifference = latest.Utc - moment;
+            double secondsDifference = Math.Abs(timeDifference.TotalSeconds);
+            double secondsPerBlock = timePerBlock.TotalSeconds;
+
+            double numberOfBlocksDifference = secondsDifference / secondsPerBlock;
+            var blockDifference = Convert.ToUInt64(numberOfBlocksDifference);
+            if (blockDifference < 1) blockDifference = 1;
+            return blockDifference;
+        }
+
+        private void EnsureRecentBlockIfNecessary(DateTime moment, TimeSpan timePerBlock)
+        {
+            var max = entries.Keys.Max();
+            var latest = entries[max];
+            var maxRetry = 10;
+            while (moment > latest.Utc)
+            {
+                var newBlock = AddCurrentBlock();
+                if (newBlock == null || newBlock.BlockNumber == latest.BlockNumber)
+                {
+                    maxRetry--;
+                    if (maxRetry == 0) throw new Exception("Unable to fetch recent block after 10x tries.");
+                    Thread.Sleep(timePerBlock);
+                }
+                max = entries.Keys.Max();
+                latest = entries[max];
+            }
+        }
+
+        private BlockTimeEntry? AddBlockNumber(decimal blockNumber)
+        {
+            return AddBlockNumber(Convert.ToUInt64(blockNumber));
+        }
+
+        private BlockTimeEntry? AddBlockNumber(ulong blockNumber)
+        {
+            if (entries.ContainsKey(blockNumber))
+            {
+                return entries[blockNumber];
+            }
+
+            if (entries.Count > MaxEntries)
+            {
+                log.Debug("Entries cleared!");
+                entries.Clear();
+                Initialize();
+            }
+
+            var time = GetTimestampFromBlock(blockNumber);
+            if (time == null)
+            {
+                log.Log("Failed to get block for number: " + blockNumber);
+                return null;
+            }
+            var entry = new BlockTimeEntry(blockNumber, time.Value);
+            log.Debug("Found block " + entry.BlockNumber + " at " + entry.Utc.ToString("o"));
+            entries.Add(blockNumber, entry);
+            return entry;
+        }
+
+        private TimeSpan EstimateTimePerBlock()
+        {
+            var min = entries.Keys.Min();
+            var max = entries.Keys.Max();
+            var clippedMin = Math.Max(max - 100, min);
+            var minTime = entries[min].Utc;
+            var clippedMinBlock = AddBlockNumber(clippedMin);
+            if (clippedMinBlock != null) minTime = clippedMinBlock.Utc;
+
+            var maxTime = entries[max].Utc;
+            var elapsedTime = maxTime - minTime;
+
+            double elapsedSeconds = elapsedTime.TotalSeconds;
+            double numberOfBlocks = max - min;
+            double secondsPerBlock = elapsedSeconds / numberOfBlocks;
+
+            var result = TimeSpan.FromSeconds(secondsPerBlock);
+            if (result.TotalSeconds < 1.0) result = TimeSpan.FromSeconds(1.0);
+            return result;
+        }
+
+        private void Initialize()
+        {
+            if (!entries.Any())
+            {
+                AddCurrentBlock();
+                AddBlockNumber(entries.Single().Key - 1);
+            }
+        }
+
+        private static void AssertMomentIsInPast(DateTime moment)
+        {
+            if (moment > DateTime.UtcNow) throw new Exception("Moment must be UTC and must be in the past.");
+        }
+
+        private BlockTimeEntry? AddCurrentBlock()
+        {
+            var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync());
+            var blockNumber = number.ToDecimal();
+            return AddBlockNumber(blockNumber);
+        }
+
+        private DateTime? GetTimestampFromBlock(ulong blockNumber)
+        {
+            try
+            {
+                var block = Time.Wait(web3.Eth.Blocks.GetBlockWithTransactionsByNumber.SendRequestAsync(new BlockParameter(blockNumber)));
+                if (block == null) return null;
+                return DateTimeOffset.FromUnixTimeSeconds(Convert.ToInt64(block.Timestamp.ToDecimal())).UtcDateTime;
+            }
+            catch (Exception ex)
+            {
+                int i = 0;
+                throw;
+            }
+        }
+
+        private BlockTimeEntry? FindClosestBeforeEntry(DateTime moment)
+        {
+            BlockTimeEntry? result = null;
+            foreach (var entry in entries.Values)
+            {
+                if (result == null)
+                {
+                    if (entry.Utc < moment) result = entry;
+                }
+                else
+                {
+                    if (entry.Utc > result.Utc && entry.Utc < moment) result = entry;
+                }
+            }
+            return result;
+        }
+
+        private BlockTimeEntry? FindClosestAfterEntry(DateTime moment)
+        {
+            BlockTimeEntry? result = null;
+            foreach (var entry in entries.Values)
+            {
+                if (result == null)
+                {
+                    if (entry.Utc > moment) result = entry;
+                }
+                else
+                {
+                    if (entry.Utc < result.Utc && entry.Utc > moment) result = entry;
+                }
+            }
+            return result;
+        }
+    }
+}

Framework/NethereumWorkflow/NethereumInteraction.cs

@@ -1,7 +1,9 @@
 using Logging;
+using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
 using Nethereum.RPC.Eth.DTOs;
 using Nethereum.Web3;
+using System.Runtime.CompilerServices;
 using Utils;
 
 namespace NethereumWorkflow
@@ -17,11 +19,12 @@ namespace NethereumWorkflow
             this.web3 = web3;
         }
 
-        public void SendEth(string toAddress, decimal ethAmount)
+        public string SendEth(string toAddress, decimal ethAmount)
         {
             log.Debug();
             var receipt = Time.Wait(web3.Eth.GetEtherTransferService().TransferEtherAndWaitForReceiptAsync(toAddress, ethAmount));
             if (!receipt.Succeeded()) throw new Exception("Unable to send Eth");
+            return receipt.TransactionHash;
         }
 
         public decimal GetEthBalance()
@@ -44,12 +47,19 @@ namespace NethereumWorkflow
             return Time.Wait(handler.QueryAsync<TResult>(contractAddress, function));
         }
 
-        public void SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
+        public string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
         {
             log.Debug();
             var handler = web3.Eth.GetContractTransactionHandler<TFunction>();
             var receipt = Time.Wait(handler.SendRequestAndWaitForReceiptAsync(contractAddress, function));
             if (!receipt.Succeeded()) throw new Exception("Unable to perform contract transaction.");
+            return receipt.TransactionHash;
+        }
+
+        public Transaction GetTransaction(string transactionHash)
+        {
+            log.Debug();
+            return Time.Wait(web3.Eth.Transactions.GetTransactionByHash.SendRequestAsync(transactionHash));
         }
 
         public decimal? GetSyncedBlockNumber()
@@ -75,5 +85,24 @@ namespace NethereumWorkflow
                 return false;
             }
         }
+
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new()
+        {
+            var blockTimeFinder = new BlockTimeFinder(web3, log);
+
+            var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From);
+            var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To);
+
+            return GetEvents<TEvent>(address, fromBlock, toBlock);
+        }
+
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new()
+        {
+            var eventHandler = web3.Eth.GetEvent<TEvent>(address);
+            var from = new BlockParameter(fromBlockNumber);
+            var to = new BlockParameter(toBlockNumber);
+            var blockFilter = Time.Wait(eventHandler.CreateFilterBlockRangeAsync(from, to));
+            return Time.Wait(eventHandler.GetAllChangesAsync(blockFilter));
+        }
     }
 }

Framework/Utils/ByteSize.cs

@@ -46,6 +46,18 @@
         }
     }
 
+    public class BytesPerSecond : ByteSize
+    {
+        public BytesPerSecond(long sizeInBytes) : base(sizeInBytes)
+        {
+        }
+
+        public override string ToString()
+        {
+            return base.ToString() + "/s";
+        }
+    }
+
     public static class ByteSizeIntExtensions
     {
         private const long Kilo = 1024;

Framework/Utils/TaskFactory.cs

@@ -1,15 +1,27 @@
-namespace ContinuousTests
+namespace Utils
 {
     public class TaskFactory
     {
         private readonly object taskLock = new();
         private readonly List<Task> activeTasks = new List<Task>();
 
-        public void Run(Action action)
+        public void Run(Action action, string name)
         {
             lock (taskLock)
             {
-                activeTasks.Add(Task.Run(action).ContinueWith(CleanupTask, null));
+                activeTasks.Add(Task.Run(() => CatchException(action, name)).ContinueWith(CleanupTask, null));
+            }
+        }
+
+        private void CatchException(Action action, string name)
+        {
+            try
+            {
+                action();
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"Exception in task '{name}': " + ex);
             }
         }
 

Framework/Utils/Time.cs

@@ -13,6 +13,11 @@
             return task.Result;
         }
 
+        public static void Wait(Task task)
+        {
+            task.Wait();
+        }
+
         public static string FormatDuration(TimeSpan d)
         {
             var result = "";
@@ -23,6 +28,35 @@
             return result;
         }
         
+        public static TimeSpan ParseTimespan(string span)
+        {
+            span = span.Replace(" ", "").Replace(",", "");
+            var result = TimeSpan.Zero;
+            var number = "";
+            foreach (var c in span)
+            {
+                if (char.IsNumber(c)) number += c;
+                else
+                {
+                    var value = Convert.ToInt32(number);
+                    number = "";
+
+                    if (c == 'd') result += TimeSpan.FromDays(value);
+                    else if (c == 'h') result += TimeSpan.FromHours(value);
+                    else if (c == 'm') result += TimeSpan.FromMinutes(value);
+                    else if (c == 's') result += TimeSpan.FromSeconds(value);
+                    else throw new Exception("Unknown time modifier: " + c);
+                }
+            }
+
+            if (!string.IsNullOrEmpty(number))
+            {
+                var value = Convert.ToInt32(number);
+                result += TimeSpan.FromSeconds(value);
+            }
+            return result;
+        }
+
         public static void WaitUntil(Func<bool> predicate)
         {
             WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1));

Framework/Utils/TimeRange.cs

@@ -0,0 +1,24 @@
+namespace Utils
+{
+    public class TimeRange
+    {
+        public TimeRange(DateTime from, DateTime to)
+        {
+            if (from < to)
+            {
+                From = from;
+                To = to;
+            }
+            else
+            {
+                From = to;
+                To = from;
+            }
+            Duration = To - From;
+        }
+
+        public DateTime From { get; }
+        public DateTime To { get; }
+        public TimeSpan Duration { get; }
+    }
+}

ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs

@@ -1,5 +1,11 @@
-using GethPlugin;
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
 using Logging;
+using Nethereum.ABI;
+using Nethereum.Hex.HexTypes;
+using Nethereum.Util;
+using NethereumWorkflow;
+using Utils;
 
 namespace CodexContractsPlugin
 {
@@ -7,10 +13,28 @@ namespace CodexContractsPlugin
     {
         CodexContractsDeployment Deployment { get; }
 
-        void MintTestTokens(IHasEthAddress owner, TestToken testTokens);
-        void MintTestTokens(EthAddress ethAddress, TestToken testTokens);
+        bool IsDeployed();
+        string MintTestTokens(IHasEthAddress owner, TestToken testTokens);
+        string MintTestTokens(EthAddress ethAddress, TestToken testTokens);
         TestToken GetTestTokenBalance(IHasEthAddress owner);
         TestToken GetTestTokenBalance(EthAddress ethAddress);
+
+        Request[] GetStorageRequests(TimeRange timeRange);
+        EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex);
+        RequestState GetRequestState(Request request);
+        RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange);
+        RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange);
+        SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange);
+        SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange);
+    }
+
+    public enum RequestState
+    {
+        New,
+        Started,
+        Cancelled,
+        Finished,
+        Failed
     }
 
     public class CodexContractsAccess : ICodexContracts
@@ -27,15 +51,19 @@ namespace CodexContractsPlugin
 
         public CodexContractsDeployment Deployment { get; }
 
-        public void MintTestTokens(IHasEthAddress owner, TestToken testTokens)
+        public bool IsDeployed()
         {
-            MintTestTokens(owner.EthAddress, testTokens);
+            return !string.IsNullOrEmpty(StartInteraction().GetTokenName(Deployment.TokenAddress));
         }
 
-        public void MintTestTokens(EthAddress ethAddress, TestToken testTokens)
+        public string MintTestTokens(IHasEthAddress owner, TestToken testTokens)
         {
-            var interaction = new ContractInteractions(log, gethNode);
-            interaction.MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
+            return MintTestTokens(owner.EthAddress, testTokens);
+        }
+
+        public string MintTestTokens(EthAddress ethAddress, TestToken testTokens)
+        {
+            return StartInteraction().MintTestTokens(ethAddress, testTokens.Amount, Deployment.TokenAddress);
         }
 
         public TestToken GetTestTokenBalance(IHasEthAddress owner)
@@ -45,9 +73,108 @@ namespace CodexContractsPlugin
 
         public TestToken GetTestTokenBalance(EthAddress ethAddress)
         {
-            var interaction = new ContractInteractions(log, gethNode);
-            var balance = interaction.GetBalance(Deployment.TokenAddress, ethAddress.Address);
+            var balance = StartInteraction().GetBalance(Deployment.TokenAddress, ethAddress.Address);
             return balance.TestTokens();
         }
+
+        public Request[] GetStorageRequests(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<StorageRequestedEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            var i = StartInteraction();
+            return events
+                    .Select(e =>
+                    {
+                        var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId);
+                        var result = requestEvent.ReturnValue1;
+                        result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                        result.RequestId = e.Event.RequestId;
+                        return result;
+                    })
+                    .ToArray();
+        }
+
+        public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<RequestFulfilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<RequestCancelledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<SlotFilledEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash);
+                return result;
+            }).ToArray();
+        }
+
+        public SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange)
+        {
+            var events = gethNode.GetEvents<SlotFreedEventDTO>(Deployment.MarketplaceAddress, timeRange);
+            return events.Select(e =>
+            {
+                var result = e.Event;
+                result.BlockNumber = e.Log.BlockNumber.ToUlong();
+                return result;
+            }).ToArray();
+        }
+
+        public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex)
+        {
+            var encoder = new ABIEncode();
+            var encoded = encoder.GetABIEncoded(
+                new ABIValue("bytes32", storageRequest.RequestId),
+                new ABIValue("uint256", slotIndex.ToBig())
+            );
+
+            var hashed = Sha3Keccack.Current.CalculateHash(encoded);
+
+            var func = new GetHostFunction
+            {
+                SlotId = hashed
+            };
+            var address = gethNode.Call<GetHostFunction, string>(Deployment.MarketplaceAddress, func);
+            if (string.IsNullOrEmpty(address)) return null;
+            return new EthAddress(address);
+        }
+
+        public RequestState GetRequestState(Request request)
+        {
+            var func = new RequestStateFunction
+            {
+                RequestId = request.RequestId
+            };
+            return gethNode.Call<RequestStateFunction, RequestState>(Deployment.MarketplaceAddress, func);
+        }
+
+        private EthAddress GetEthAddressFromTransaction(string transactionHash)
+        {
+            var transaction = gethNode.GetTransaction(transactionHash);
+            return new EthAddress(transaction.From);
+        }
+
+        private ContractInteractions StartInteraction()
+        {
+            return new ContractInteractions(log, gethNode);
+        }
     }
 }

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -7,7 +7,7 @@ namespace CodexContractsPlugin
 {
     public class CodexContractsContainerRecipe : ContainerRecipeFactory
     {
-        public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests";
+        public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:sha-b5f3399-dist-tests";
 
         public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
         public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
@@ -21,6 +21,8 @@ namespace CodexContractsPlugin
 
             var address = config.GethNode.StartResult.Container.GetAddress(new NullLog(), GethContainerRecipe.HttpPortTag);
 
+            SetSchedulingAffinity(notIn: "false");
+
             AddEnvVar("DISTTEST_NETWORK_URL", address.ToString());
             AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork");
             AddEnvVar("KEEP_ALIVE", "1");

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -85,7 +85,7 @@ namespace CodexContractsPlugin
 
         private void WaitUntil(Func<bool> predicate)
         {
-            Time.WaitUntil(predicate, TimeSpan.FromMinutes(3), TimeSpan.FromSeconds(2));
+            Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2));
         }
 
         private StartupConfig CreateStartupConfig(IGethNode gethNode)

ProjectPlugins/CodexContractsPlugin/ContractInteractions.cs

@@ -1,7 +1,9 @@
-using GethPlugin;
+using CodexContractsPlugin.Marketplace;
+using GethPlugin;
 using Logging;
 using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
+using Nethereum.Hex.HexConvertors.Extensions;
 using NethereumWorkflow;
 using System.Numerics;
 
@@ -26,10 +28,26 @@ namespace CodexContractsPlugin
             return gethNode.Call<GetTokenFunction, string>(marketplaceAddress, function);
         }
 
-        public void MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
+        public string GetTokenName(string tokenAddress)
+        {
+            try
+            {
+                log.Debug(tokenAddress);
+                var function = new GetTokenNameFunction();
+
+                return gethNode.Call<GetTokenNameFunction, string>(tokenAddress, function);
+            }
+            catch (Exception ex)
+            {
+                log.Log("Failed to get token name: " + ex);
+                return string.Empty;
+            }
+        }
+
+        public string MintTestTokens(EthAddress address, decimal amount, string tokenAddress)
         {
             log.Debug($"{amount} -> {address} (token: {tokenAddress})");
-            MintTokens(address.Address, amount, tokenAddress);
+            return MintTokens(address.Address, amount, tokenAddress);
         }
 
         public decimal GetBalance(string tokenAddress, string account)
@@ -43,6 +61,17 @@ namespace CodexContractsPlugin
             return gethNode.Call<GetTokenBalanceFunction, BigInteger>(tokenAddress, function).ToDecimal();
         }
 
+        public GetRequestOutputDTO GetRequest(string marketplaceAddress, byte[] requestId)
+        {
+
+            log.Debug($"({marketplaceAddress}) {requestId.ToHex(true)}");
+            var func = new GetRequestFunction
+            {
+                RequestId = requestId
+            };
+            return gethNode.Call<GetRequestFunction, GetRequestOutputDTO>(marketplaceAddress, func);
+        }
+
         public bool IsSynced(string marketplaceAddress, string marketplaceAbi)
         {
             log.Debug();
@@ -56,7 +85,7 @@ namespace CodexContractsPlugin
             }
         }
 
-        private void MintTokens(string account, decimal amount, string tokenAddress)
+        private string MintTokens(string account, decimal amount, string tokenAddress)
         {
             log.Debug($"({tokenAddress}) {amount} --> {account}");
             if (string.IsNullOrEmpty(account)) throw new ArgumentException("Invalid arguments for MintTestTokens");
@@ -67,7 +96,7 @@ namespace CodexContractsPlugin
                 Amount = amount.ToBig()
             };
 
-            gethNode.SendTransaction(tokenAddress, function);
+            return gethNode.SendTransaction(tokenAddress, function);
         }
 
         private bool IsBlockNumberOK()
@@ -87,6 +116,11 @@ namespace CodexContractsPlugin
     {
     }
 
+    [Function("name", "string")]
+    public class GetTokenNameFunction : FunctionMessage
+    {
+    }
+
     [Function("mint")]
     public class MintTokensFunction : FunctionMessage
     {

ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs

@@ -0,0 +1,35 @@
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
+using GethPlugin;
+
+namespace CodexContractsPlugin.Marketplace
+{
+    public partial class Request : RequestBase
+    {
+        public ulong BlockNumber { get; set; }
+        public byte[] RequestId { get; set; }
+
+        public EthAddress ClientAddress { get { return new EthAddress(Client); } }
+    }
+
+    public partial class RequestFulfilledEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+    }
+
+    public partial class RequestCancelledEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+    }
+
+    public partial class SlotFilledEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+        public EthAddress Host { get; set; }
+    }
+
+    public partial class SlotFreedEventDTO
+    {
+        public ulong BlockNumber { get; set; }
+    }
+}
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexContractsPlugin/Marketplace/Marketplace.cs

@@ -0,0 +1,517 @@
+using Nethereum.ABI.FunctionEncoding.Attributes;
+using Nethereum.Contracts;
+using System.Numerics;
+
+// Generated code, do not modify.
+
+#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.
+namespace CodexContractsPlugin.Marketplace
+{
+    public partial class ConfigFunction : ConfigFunctionBase { }
+
+    [Function("config", typeof(ConfigOutputDTO))]
+    public class ConfigFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class FillSlotFunction : FillSlotFunctionBase { }
+
+    [Function("fillSlot")]
+    public class FillSlotFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("uint256", "slotIndex", 2)]
+        public virtual BigInteger SlotIndex { get; set; }
+        [Parameter("bytes", "proof", 3)]
+        public virtual byte[] Proof { get; set; }
+    }
+
+    public partial class FreeSlotFunction : FreeSlotFunctionBase { }
+
+    [Function("freeSlot")]
+    public class FreeSlotFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class GetActiveSlotFunction : GetActiveSlotFunctionBase { }
+
+    [Function("getActiveSlot", typeof(GetActiveSlotOutputDTO))]
+    public class GetActiveSlotFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class GetChallengeFunction : GetChallengeFunctionBase { }
+
+    [Function("getChallenge", "bytes32")]
+    public class GetChallengeFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class GetHostFunction : GetHostFunctionBase { }
+
+    [Function("getHost", "address")]
+    public class GetHostFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class GetPointerFunction : GetPointerFunctionBase { }
+
+    [Function("getPointer", "uint8")]
+    public class GetPointerFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class GetRequestFunction : GetRequestFunctionBase { }
+
+    [Function("getRequest", typeof(GetRequestOutputDTO))]
+    public class GetRequestFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class IsProofRequiredFunction : IsProofRequiredFunctionBase { }
+
+    [Function("isProofRequired", "bool")]
+    public class IsProofRequiredFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class MarkProofAsMissingFunction : MarkProofAsMissingFunctionBase { }
+
+    [Function("markProofAsMissing")]
+    public class MarkProofAsMissingFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+        [Parameter("uint256", "period", 2)]
+        public virtual BigInteger Period { get; set; }
+    }
+
+    public partial class MissingProofsFunction : MissingProofsFunctionBase { }
+
+    [Function("missingProofs", "uint256")]
+    public class MissingProofsFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class MyRequestsFunction : MyRequestsFunctionBase { }
+
+    [Function("myRequests", "bytes32[]")]
+    public class MyRequestsFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class MySlotsFunction : MySlotsFunctionBase { }
+
+    [Function("mySlots", "bytes32[]")]
+    public class MySlotsFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class RequestEndFunction : RequestEndFunctionBase { }
+
+    [Function("requestEnd", "uint256")]
+    public class RequestEndFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestStateFunction : RequestStateFunctionBase { }
+
+    [Function("requestState", "uint8")]
+    public class RequestStateFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestStorageFunction : RequestStorageFunctionBase { }
+
+    [Function("requestStorage")]
+    public class RequestStorageFunctionBase : FunctionMessage
+    {
+        [Parameter("tuple", "request", 1)]
+        public virtual Request Request { get; set; }
+    }
+
+    public partial class SlotStateFunction : SlotStateFunctionBase { }
+
+    [Function("slotState", "uint8")]
+    public class SlotStateFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "slotId", 1)]
+        public virtual byte[] SlotId { get; set; }
+    }
+
+    public partial class SubmitProofFunction : SubmitProofFunctionBase { }
+
+    [Function("submitProof")]
+    public class SubmitProofFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+        [Parameter("bytes", "proof", 2)]
+        public virtual byte[] Proof { get; set; }
+    }
+
+    public partial class TokenFunction : TokenFunctionBase { }
+
+    [Function("token", "address")]
+    public class TokenFunctionBase : FunctionMessage
+    {
+
+    }
+
+    public partial class WillProofBeRequiredFunction : WillProofBeRequiredFunctionBase { }
+
+    [Function("willProofBeRequired", "bool")]
+    public class WillProofBeRequiredFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "id", 1)]
+        public virtual byte[] Id { get; set; }
+    }
+
+    public partial class WithdrawFundsFunction : WithdrawFundsFunctionBase { }
+
+    [Function("withdrawFunds")]
+    public class WithdrawFundsFunctionBase : FunctionMessage
+    {
+        [Parameter("bytes32", "requestId", 1)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class ProofSubmittedEventDTO : ProofSubmittedEventDTOBase { }
+
+    [Event("ProofSubmitted")]
+    public class ProofSubmittedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "id", 1, false)]
+        public virtual byte[] Id { get; set; }
+        [Parameter("bytes", "proof", 2, false)]
+        public virtual byte[] Proof { get; set; }
+    }
+
+    public partial class RequestCancelledEventDTO : RequestCancelledEventDTOBase { }
+
+    [Event("RequestCancelled")]
+    public class RequestCancelledEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestFailedEventDTO : RequestFailedEventDTOBase { }
+
+    [Event("RequestFailed")]
+    public class RequestFailedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class RequestFulfilledEventDTO : RequestFulfilledEventDTOBase { }
+
+    [Event("RequestFulfilled")]
+    public class RequestFulfilledEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+    }
+
+    public partial class SlotFilledEventDTO : SlotFilledEventDTOBase { }
+
+    [Event("SlotFilled")]
+    public class SlotFilledEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("uint256", "slotIndex", 2, false)]
+        public virtual BigInteger SlotIndex { get; set; }
+    }
+
+    public partial class SlotFreedEventDTO : SlotFreedEventDTOBase { }
+
+    [Event("SlotFreed")]
+    public class SlotFreedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, true)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("uint256", "slotIndex", 2, false)]
+        public virtual BigInteger SlotIndex { get; set; }
+    }
+
+    public partial class StorageRequestedEventDTO : StorageRequestedEventDTOBase { }
+
+    [Event("StorageRequested")]
+    public class StorageRequestedEventDTOBase : IEventDTO
+    {
+        [Parameter("bytes32", "requestId", 1, false)]
+        public virtual byte[] RequestId { get; set; }
+        [Parameter("tuple", "ask", 2, false)]
+        public virtual Ask Ask { get; set; }
+        [Parameter("uint256", "expiry", 3, false)]
+        public virtual BigInteger Expiry { get; set; }
+    }
+
+    public partial class ConfigOutputDTO : ConfigOutputDTOBase { }
+
+    [FunctionOutput]
+    public class ConfigOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("tuple", "collateral", 1)]
+        public virtual CollateralConfig Collateral { get; set; }
+        [Parameter("tuple", "proofs", 2)]
+        public virtual ProofConfig Proofs { get; set; }
+    }
+
+
+
+
+
+    public partial class GetActiveSlotOutputDTO : GetActiveSlotOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetActiveSlotOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("tuple", "", 1)]
+        public virtual ActiveSlot ReturnValue1 { get; set; }
+    }
+
+    public partial class GetChallengeOutputDTO : GetChallengeOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetChallengeOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bytes32", "", 1)]
+        public virtual byte[] ReturnValue1 { get; set; }
+    }
+
+    public partial class GetHostOutputDTO : GetHostOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetHostOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("address", "", 1)]
+        public virtual string ReturnValue1 { get; set; }
+    }
+
+    public partial class GetPointerOutputDTO : GetPointerOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetPointerOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint8", "", 1)]
+        public virtual byte ReturnValue1 { get; set; }
+    }
+
+    public partial class GetRequestOutputDTO : GetRequestOutputDTOBase { }
+
+    [FunctionOutput]
+    public class GetRequestOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("tuple", "", 1)]
+        public virtual Request ReturnValue1 { get; set; }
+    }
+
+    public partial class IsProofRequiredOutputDTO : IsProofRequiredOutputDTOBase { }
+
+    [FunctionOutput]
+    public class IsProofRequiredOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bool", "", 1)]
+        public virtual bool ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class MissingProofsOutputDTO : MissingProofsOutputDTOBase { }
+
+    [FunctionOutput]
+    public class MissingProofsOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint256", "", 1)]
+        public virtual BigInteger ReturnValue1 { get; set; }
+    }
+
+    public partial class MyRequestsOutputDTO : MyRequestsOutputDTOBase { }
+
+    [FunctionOutput]
+    public class MyRequestsOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bytes32[]", "", 1)]
+        public virtual List<byte[]> ReturnValue1 { get; set; }
+    }
+
+    public partial class MySlotsOutputDTO : MySlotsOutputDTOBase { }
+
+    [FunctionOutput]
+    public class MySlotsOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bytes32[]", "", 1)]
+        public virtual List<byte[]> ReturnValue1 { get; set; }
+    }
+
+    public partial class RequestEndOutputDTO : RequestEndOutputDTOBase { }
+
+    [FunctionOutput]
+    public class RequestEndOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint256", "", 1)]
+        public virtual BigInteger ReturnValue1 { get; set; }
+    }
+
+    public partial class RequestStateOutputDTO : RequestStateOutputDTOBase { }
+
+    [FunctionOutput]
+    public class RequestStateOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint8", "", 1)]
+        public virtual byte ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class SlotStateOutputDTO : SlotStateOutputDTOBase { }
+
+    [FunctionOutput]
+    public class SlotStateOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("uint8", "", 1)]
+        public virtual byte ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class TokenOutputDTO : TokenOutputDTOBase { }
+
+    [FunctionOutput]
+    public class TokenOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("address", "", 1)]
+        public virtual string ReturnValue1 { get; set; }
+    }
+
+    public partial class WillProofBeRequiredOutputDTO : WillProofBeRequiredOutputDTOBase { }
+
+    [FunctionOutput]
+    public class WillProofBeRequiredOutputDTOBase : IFunctionOutputDTO
+    {
+        [Parameter("bool", "", 1)]
+        public virtual bool ReturnValue1 { get; set; }
+    }
+
+
+
+    public partial class CollateralConfig : CollateralConfigBase { }
+
+    public class CollateralConfigBase
+    {
+        [Parameter("uint8", "repairRewardPercentage", 1)]
+        public virtual byte RepairRewardPercentage { get; set; }
+        [Parameter("uint8", "maxNumberOfSlashes", 2)]
+        public virtual byte MaxNumberOfSlashes { get; set; }
+        [Parameter("uint16", "slashCriterion", 3)]
+        public virtual ushort SlashCriterion { get; set; }
+        [Parameter("uint8", "slashPercentage", 4)]
+        public virtual byte SlashPercentage { get; set; }
+    }
+
+    public partial class ProofConfig : ProofConfigBase { }
+
+    public class ProofConfigBase
+    {
+        [Parameter("uint256", "period", 1)]
+        public virtual BigInteger Period { get; set; }
+        [Parameter("uint256", "timeout", 2)]
+        public virtual BigInteger Timeout { get; set; }
+        [Parameter("uint8", "downtime", 3)]
+        public virtual byte Downtime { get; set; }
+    }
+
+    public partial class MarketplaceConfig : MarketplaceConfigBase { }
+
+    public class MarketplaceConfigBase
+    {
+        [Parameter("tuple", "collateral", 1)]
+        public virtual CollateralConfig Collateral { get; set; }
+        [Parameter("tuple", "proofs", 2)]
+        public virtual ProofConfig Proofs { get; set; }
+    }
+
+    public partial class Ask : AskBase { }
+
+    public class AskBase
+    {
+        [Parameter("uint64", "slots", 1)]
+        public virtual ulong Slots { get; set; }
+        [Parameter("uint256", "slotSize", 2)]
+        public virtual BigInteger SlotSize { get; set; }
+        [Parameter("uint256", "duration", 3)]
+        public virtual BigInteger Duration { get; set; }
+        [Parameter("uint256", "proofProbability", 4)]
+        public virtual BigInteger ProofProbability { get; set; }
+        [Parameter("uint256", "reward", 5)]
+        public virtual BigInteger Reward { get; set; }
+        [Parameter("uint256", "collateral", 6)]
+        public virtual BigInteger Collateral { get; set; }
+        [Parameter("uint64", "maxSlotLoss", 7)]
+        public virtual ulong MaxSlotLoss { get; set; }
+    }
+
+    public partial class Content : ContentBase { }
+
+    public class ContentBase
+    {
+        [Parameter("string", "cid", 1)]
+        public virtual string Cid { get; set; }
+        [Parameter("bytes32", "merkleRoot", 2)]
+        public virtual byte[] MerkleRoot { get; set; }
+    }
+
+    public partial class Request : RequestBase { }
+
+    public class RequestBase
+    {
+        [Parameter("address", "client", 1)]
+        public virtual string Client { get; set; }
+        [Parameter("tuple", "ask", 2)]
+        public virtual Ask Ask { get; set; }
+        [Parameter("tuple", "content", 3)]
+        public virtual Content Content { get; set; }
+        [Parameter("uint256", "expiry", 4)]
+        public virtual BigInteger Expiry { get; set; }
+        [Parameter("bytes32", "nonce", 5)]
+        public virtual byte[] Nonce { get; set; }
+    }
+
+    public partial class ActiveSlot : ActiveSlotBase { }
+
+    public class ActiveSlotBase
+    {
+        [Parameter("tuple", "request", 1)]
+        public virtual Request Request { get; set; }
+        [Parameter("uint256", "slotIndex", 2)]
+        public virtual BigInteger SlotIndex { get; set; }
+    }
+}
+#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider declaring as nullable.

ProjectPlugins/CodexContractsPlugin/Marketplace/README.md

@@ -0,0 +1 @@
+This code was generated using the Nethereum code generator, here: http://playground.nethereum.com

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs

@@ -13,16 +13,30 @@ namespace CodexDiscordBotPlugin
         {
             var config = startupConfig.Get<DiscordBotStartupConfig>();
 
+            SetSchedulingAffinity(notIn: "false");
+
             AddEnvVar("TOKEN", config.Token);
             AddEnvVar("SERVERNAME", config.ServerName);
             AddEnvVar("ADMINROLE", config.AdminRoleName);
             AddEnvVar("ADMINCHANNELNAME", config.AdminChannelName);
+            AddEnvVar("KUBECONFIG", "/opt/kubeconfig.yaml");
+            AddEnvVar("KUBENAMESPACE", config.KubeNamespace);
+
+            var gethInfo = config.GethInfo;
+            AddEnvVar("GETH_HOST", gethInfo.Host);
+            AddEnvVar("GETH_HTTP_PORT", gethInfo.Port.ToString());
+            AddEnvVar("GETH_PRIVATE_KEY", gethInfo.PrivKey);
+            AddEnvVar("CODEXCONTRACTS_MARKETPLACEADDRESS", gethInfo.MarketplaceAddress);
+            AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress);
+            AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi);
 
             if (!string.IsNullOrEmpty(config.DataPath))
             {
                 AddEnvVar("DATAPATH", config.DataPath);
                 AddVolume(config.DataPath, 1.GB());
             }
+
+            AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml", secret: "discordbot-sa-kubeconfig");
         }
     }
 }

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs

@@ -2,13 +2,15 @@
 {
     public class DiscordBotStartupConfig
     {
-        public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName)
+        public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName, string kubeNamespace, DiscordBotGethInfo gethInfo)
         {
             Name = name;
             Token = token;
             ServerName = serverName;
             AdminRoleName = adminRoleName;
             AdminChannelName = adminChannelName;
+            KubeNamespace = kubeNamespace;
+            GethInfo = gethInfo;
         }
 
         public string Name { get; }
@@ -16,6 +18,28 @@
         public string ServerName { get; }
         public string AdminRoleName { get; }
         public string AdminChannelName { get; }
+        public string KubeNamespace { get; }
+        public DiscordBotGethInfo GethInfo { get; }
         public string? DataPath { get; set; }
     }
+
+    public class DiscordBotGethInfo
+    {
+        public DiscordBotGethInfo(string host, int port, string privKey, string marketplaceAddress, string tokenAddress, string abi)
+        {
+            Host = host;
+            Port = port;
+            PrivKey = privKey;
+            MarketplaceAddress = marketplaceAddress;
+            TokenAddress = tokenAddress;
+            Abi = abi;
+        }
+
+        public string Host { get; }
+        public int Port { get; }
+        public string PrivKey { get; }
+        public string MarketplaceAddress { get; }
+        public string TokenAddress { get; }
+        public string Abi { get; }
+    }
 }

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -68,12 +68,12 @@ namespace CodexPlugin
 
         public Stream DownloadFile(string contentId)
         {
-            return Http().HttpGetStream("data/" + contentId);
+            return Http().HttpGetStream("data/" + contentId + "/network");
         }
 
         public CodexLocalDataResponse[] LocalFiles()
         {
-            return Http().HttpGetJson<CodexLocalDataResponse[]>("local");
+            return Http().HttpGetJson<CodexLocalDataResponse[]>("data");
         }
 
         public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)

ProjectPlugins/CodexPlugin/CodexApiTypes.cs

@@ -5,9 +5,10 @@ namespace CodexPlugin
     public class CodexDebugResponse
     {
         public string id { get; set; } = string.Empty;
-        public string[] addrs { get; set; } = new string[0];
+        public string[] addrs { get; set; } = Array.Empty<string>();
         public string repo { get; set; } = string.Empty;
         public string spr { get; set; } = string.Empty;
+        public string[] announceAddresses { get; set; } = Array.Empty<string>();
         public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
         public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
         public CodexDebugVersionResponse codex { get; set; } = new();

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -29,6 +29,9 @@ namespace CodexPlugin
             SetResourcesRequest(milliCPUs: 100, memory: 100.MB());
             //SetResourceLimits(milliCPUs: 4000, memory: 12.GB());
 
+            SetSchedulingAffinity(notIn: "false");
+            SetSystemCriticalPriority();
+
             var config = startupConfig.Get<CodexStartupConfig>();
 
             var apiPort = CreateApiPort(config, ApiPortTag);
@@ -45,8 +48,9 @@ namespace CodexPlugin
 
             if (config.PublicTestNet != null)
             {
-                AddEnvVar("CODEX_NAT", config.PublicTestNet.PublicNatIP);
+                // This makes the node announce itself to its public IP address.
                 AddEnvVar("NAT_IP_AUTO", "false");
+                AddEnvVar("NAT_PUBLIC_IP_AUTO", PublicIpService.Address);
             }
             else
             {

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -6,8 +6,12 @@ namespace CodexPlugin
 {
     public class CodexDeployment
     {
-        public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment, CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer, RunningContainers? discordBotContainer, DeploymentMetadata metadata)
+        public CodexDeployment(CodexInstance[] codexInstances, GethDeployment gethDeployment,
+            CodexContractsDeployment codexContractsDeployment, RunningContainers? prometheusContainer,
+            RunningContainers? discordBotContainer, DeploymentMetadata metadata,
+            String id)
         {
+            Id = id;
             CodexInstances = codexInstances;
             GethDeployment = gethDeployment;
             CodexContractsDeployment = codexContractsDeployment;
@@ -16,6 +20,7 @@ namespace CodexPlugin
             Metadata = metadata;
         }
 
+        public String Id { get; }
         public CodexInstance[] CodexInstances { get; }
         public GethDeployment GethDeployment { get; }
         public CodexContractsDeployment CodexContractsDeployment { get; }
@@ -38,7 +43,10 @@ namespace CodexPlugin
 
     public class DeploymentMetadata
     {
-        public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
+        public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace,
+            int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel,
+            int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI,
+            int blockMN)
         {
             Name = name;
             StartUtc = startUtc;
@@ -73,4 +81,4 @@ namespace CodexPlugin
         public int BlockMI { get; }
         public int BlockMN { get; }
     }
-}
+}

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -25,6 +25,7 @@ namespace CodexPlugin
         IMarketplaceAccess Marketplace { get; }
         CrashWatcher CrashWatcher { get; }
         PodInfo GetPodInfo();
+        ITransferSpeeds TransferSpeeds { get; }
         void Stop();
     }
 
@@ -34,6 +35,7 @@ namespace CodexPlugin
         private const string UploadFailedMessage = "Unable to store block";
         private readonly IPluginTools tools;
         private readonly EthAddress? ethAddress;
+        private readonly TransferSpeeds transferSpeeds;
 
         public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAddress? ethAddress)
         {
@@ -43,6 +45,7 @@ namespace CodexPlugin
             Group = group;
             Marketplace = marketplaceAccess;
             Version = new CodexDebugVersionResponse();
+            transferSpeeds = new TransferSpeeds();
         }
 
         public RunningContainer Container { get { return CodexAccess.Container; } }
@@ -51,6 +54,7 @@ namespace CodexPlugin
         public CodexNodeGroup Group { get; }
         public IMarketplaceAccess Marketplace { get; }
         public CodexDebugVersionResponse Version { get; private set; }
+        public ITransferSpeeds TransferSpeeds { get => transferSpeeds; }
         public IMetricsScrapeTarget MetricsScrapeTarget
         {
             get
@@ -101,11 +105,14 @@ namespace CodexPlugin
 
             var logMessage = $"Uploading file {file.Describe()}...";
             Log(logMessage);
-            var response = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
+            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () =>
             {
                 return CodexAccess.UploadFile(fileStream);
             });
 
+            var response = measurement.Value;
+            transferSpeeds.AddUploadSample(file.GetFilesize(), measurement.Duration);
+
             if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response.");
             if (response.StartsWith(UploadFailedMessage)) FrameworkAssert.Fail("Node failed to store block.");
 
@@ -118,7 +125,8 @@ namespace CodexPlugin
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
             Log(logMessage);
             var file = tools.GetFileManager().CreateEmptyFile(fileLabel);
-            Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file));
+            var measurement = Stopwatch.Measure(tools.GetLog(), logMessage, () => DownloadToFile(contentId.Id, file));
+            transferSpeeds.AddDownloadSample(file.GetFilesize(), measurement);
             Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
             return file;
         }

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -19,12 +19,13 @@ namespace CodexPlugin
 
         public void Announce()
         {
-            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}'");
+            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
         }
 
         public void AddMetadata(IAddMetadata metadata)
         {
             metadata.Add("codexid", codexStarter.GetCodexId());
+            metadata.Add("codexrevision", codexStarter.GetCodexRevision());
         }
 
         public void Decommission()

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -27,6 +27,13 @@ namespace CodexPlugin
 
     public class CodexLogCustomTopics
     {
+        public CodexLogCustomTopics(CodexLogLevel discV5, CodexLogLevel libp2p, CodexLogLevel blockExchange)
+        {
+            DiscV5 = discV5;
+            Libp2p = libp2p;
+            BlockExchange = blockExchange;
+        }
+
         public CodexLogCustomTopics(CodexLogLevel discV5, CodexLogLevel libp2p)
         {
             DiscV5 = discV5;
@@ -35,6 +42,7 @@ namespace CodexPlugin
 
         public CodexLogLevel DiscV5 { get; set; }
         public CodexLogLevel Libp2p { get; set; }
+        public CodexLogLevel? BlockExchange { get; }
     }
 
     public class CodexSetup : CodexStartupConfig, ICodexSetup
@@ -133,7 +141,7 @@ namespace CodexPlugin
 
         private IEnumerable<string> DescribeArgs()
         {
-            if (PublicTestNet != null) yield return $"<!>Public TestNet at {PublicTestNet.PublicNatIP}:{PublicTestNet.PublicListenPort}<!>";
+            if (PublicTestNet != null) yield return $"<!>Public TestNet with listenPort: {PublicTestNet.PublicListenPort}<!>";
             yield return $"LogLevel={LogLevelWithTopics()}";
             if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
             if (StorageQuota != null) yield return $"StorageQuota={StorageQuota}";

ProjectPlugins/CodexPlugin/CodexStarter.cs

@@ -66,6 +66,12 @@ namespace CodexPlugin
             return recipe.Image;
         }
 
+        public string GetCodexRevision()
+        {
+            if (versionResponse != null) return versionResponse.revision;
+            return "unknown";
+        }
+
         private StartupConfig CreateStartupConfig(CodexSetup codexSetup)
         {
             var startupConfig = new StartupConfig();

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -52,12 +52,31 @@ namespace CodexPlugin
                     "connection",
                     "connmanager",
                     "websock",
-                    "ws-session"
+                    "ws-session",
+                    "dialer",
+                    "muxedupgrade",
+                    "upgrade",
+                    "identify"
+                };
+                var blockExchangeTopics = new[]
+                {
+                    "codex",
+                    "pendingblocks",
+                    "peerctxstore",
+                    "discoveryengine",
+                    "blockexcengine",
+                    "blockexcnetwork",
+                    "blockexcnetworkpeer"
                 };
 
                 level = $"{level};" +
                     $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" +
                     $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)}";
+
+                if (CustomTopics.BlockExchange != null)
+                {
+                    level += $";{CustomTopics.BlockExchange.ToString()!.ToLowerInvariant()}:{string.Join(",", blockExchangeTopics)}";
+                }
             }
             return level;
         }
@@ -65,7 +84,6 @@ namespace CodexPlugin
 
     public class CodexTestNetConfig
     {
-        public string PublicNatIP { get; set; } = string.Empty;
         public int PublicDiscoveryPort { get; set; }
         public int PublicListenPort { get; set; }
     }

ProjectPlugins/CodexPlugin/MarketplaceAccess.cs

@@ -10,6 +10,7 @@ namespace CodexPlugin
     {
         string MakeStorageAvailable(ByteSize size, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration);
         StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
+        StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, TimeSpan expiry);
     }
 
     public class MarketplaceAccess : IMarketplaceAccess
@@ -25,13 +26,20 @@ namespace CodexPlugin
 
         public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
         {
+            return RequestStorage(contentId, pricePerSlotPerSecond, requiredCollateral, minRequiredNumberOfNodes, proofProbability, duration, duration / 2);
+        }
+
+        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, TimeSpan expiry)
+        {
+            var expireUtc = DateTimeOffset.UtcNow.ToUnixTimeSeconds() + expiry.TotalSeconds;
+
             var request = new CodexSalesRequestStorageRequest
             {
                 duration = ToDecInt(duration.TotalSeconds),
                 proofProbability = ToDecInt(proofProbability),
                 reward = ToDecInt(pricePerSlotPerSecond),
                 collateral = ToDecInt(requiredCollateral),
-                expiry = null,
+                expiry = ToDecInt(expireUtc),
                 nodes = minRequiredNumberOfNodes,
                 tolerance = null,
             };
@@ -41,11 +49,15 @@ namespace CodexPlugin
                 $"requiredCollateral: {requiredCollateral}, " +
                 $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
                 $"proofProbability: {proofProbability}, " +
+                $"expiry: {Time.FormatDuration(expiry)}, " +
                 $"duration: {Time.FormatDuration(duration)})");
 
             var response = codexAccess.RequestStorage(request, contentId.Id);
 
-            if (response == "Purchasing not available")
+            if (response == "Purchasing not available" || 
+                response == "Expiry required" ||
+                response == "Expiry needs to be in future" ||
+                response == "Expiry has to be before the request's end (now + duration)")
             {
                 throw new InvalidOperationException(response);
             }
@@ -104,6 +116,12 @@ namespace CodexPlugin
             return null!;
         }
 
+        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration, TimeSpan expiry)
+        {
+            Unavailable();
+            throw new NotImplementedException();
+        }
+
         public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration)
         {
             Unavailable();

ProjectPlugins/CodexPlugin/TransferSpeeds.cs

@@ -0,0 +1,67 @@
+using Utils;
+
+namespace CodexPlugin
+{
+    public interface ITransferSpeeds
+    {
+        BytesPerSecond? GetUploadSpeed();
+        BytesPerSecond? GetDownloadSpeed();
+    }
+
+    public class TransferSpeeds : ITransferSpeeds
+    {
+        private readonly List<BytesPerSecond> uploads = new List<BytesPerSecond>();
+        private readonly List<BytesPerSecond> downloads = new List<BytesPerSecond>();
+
+        public void AddUploadSample(ByteSize bytes, TimeSpan duration)
+        {
+            uploads.Add(Convert(bytes, duration));
+        }
+
+        public void AddDownloadSample(ByteSize bytes, TimeSpan duration)
+        {
+            downloads.Add(Convert(bytes, duration));
+        }
+
+        public BytesPerSecond? GetUploadSpeed()
+        {
+            if (!uploads.Any()) return null;
+            return uploads.Average();
+        }
+
+        public BytesPerSecond? GetDownloadSpeed()
+        {
+            if (!downloads.Any()) return null;
+            return downloads.Average();
+        }
+
+        private static BytesPerSecond Convert(ByteSize size, TimeSpan duration)
+        {
+            double bytes = size.SizeInBytes;
+            double seconds = duration.TotalSeconds;
+
+            return new BytesPerSecond(System.Convert.ToInt64(Math.Round(bytes / seconds)));
+        }
+    }
+
+    public static class ListExtensions
+    {
+        public static BytesPerSecond Average(this List<BytesPerSecond> list)
+        {
+            double sum = list.Sum(i => i.SizeInBytes);
+            double num = list.Count;
+
+            return new BytesPerSecond(Convert.ToInt64(Math.Round(sum / num)));
+        }
+
+        public static BytesPerSecond? OptionalAverage(this List<BytesPerSecond?>? list)
+        {
+            if (list == null || !list.Any() || !list.Any(i => i != null)) return null;
+            var values = list.Where(i => i != null).Cast<BytesPerSecond>().ToArray();
+            double sum = values.Sum(i => i.SizeInBytes);
+            double num = values.Length;
+
+            return new BytesPerSecond(Convert.ToInt64(Math.Round(sum / num)));
+        }
+    }
+}

ProjectPlugins/GethPlugin/EthAddress.cs

@@ -9,11 +9,22 @@
     {
         public EthAddress(string address)
         {
-            Address = address;
+            Address = address.ToLowerInvariant();
         }
 
         public string Address { get; }
 
+        public override bool Equals(object? obj)
+        {
+            return obj is EthAddress address &&
+                   Address == address.Address;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Address);
+        }
+
         public override string ToString()
         {
             return Address;

ProjectPlugins/GethPlugin/GethContainerRecipe.cs

@@ -24,13 +24,19 @@ namespace GethPlugin
 
             var args = CreateArgs(config);
 
+            SetSchedulingAffinity(notIn: "false");
+            SetSystemCriticalPriority();
+
             AddEnvVar("GETH_ARGS", args);
         }
 
         private string CreateArgs(GethStartupConfig config)
         {
-            if (config.IsMiner) AddEnvVar("ENABLE_MINER", "1");
-            UnlockAccounts(0, 1);
+            if (config.IsMiner)
+            {
+                AddEnvVar("ENABLE_MINER", "1");
+                UnlockAccounts(0, 1);
+            }
 
             var httpPort = CreateApiPort(config, tag: HttpPortTag);
             var discovery = CreateDiscoveryPort(config);
@@ -38,16 +44,20 @@ namespace GethPlugin
             var authRpc = CreateP2pPort(config, tag: AuthRpcPortTag);
             var wsPort = CreateP2pPort(config, tag: WsPortTag);
 
-            var args = $"--http.addr 0.0.0.0 --http.port {httpPort.Number} --port {listen.Number} --discovery.port {discovery.Number} {GetTestNetArgs(config)} {defaultArgs}";
+            var args = $"--http.addr 0.0.0.0 --http.port {httpPort.Number} --port {listen.Number} --discovery.port {discovery.Number} {defaultArgs}";
 
             if (config.BootstrapNode != null)
             {
                 var bootPubKey = config.BootstrapNode.PublicKey;
                 var bootIp = config.BootstrapNode.IpAddress;
                 var bootPort = config.BootstrapNode.Port;
-                var bootstrapArg = $" --bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}";
+                var bootstrapArg = $" --bootnodes enode://{bootPubKey}@{bootIp}:{bootPort}";
                 args += bootstrapArg;
             }
+            if (config.IsPublicTestNet != null)
+            {
+                AddEnvVar("NAT_PUBLIC_IP_AUTO", PublicIpService.Address);
+            }
 
             return args + $" --authrpc.port {authRpc.Number} --ws --ws.addr 0.0.0.0 --ws.port {wsPort.Number}";
         }
@@ -55,20 +65,13 @@ namespace GethPlugin
         private void UnlockAccounts(int startIndex, int numberOfAccounts)
         {
             if (startIndex < 0) throw new ArgumentException();
-            if (numberOfAccounts < 1) throw new ArgumentException();
+            if (numberOfAccounts < 0) throw new ArgumentException();
             if (startIndex + numberOfAccounts > 1000) throw new ArgumentException("Out of accounts!");
 
             AddEnvVar("UNLOCK_START_INDEX", startIndex.ToString());
             AddEnvVar("UNLOCK_NUMBER", numberOfAccounts.ToString());
         }
 
-        private string GetTestNetArgs(GethStartupConfig config)
-        {
-            if (config.IsPublicTestNet == null) return string.Empty;
-
-            return $"--nat=extip:{config.IsPublicTestNet.PublicIp}";
-        }
-
         private Port CreateDiscoveryPort(GethStartupConfig config)
         {
             if (config.IsPublicTestNet == null) return AddInternalPort(DiscoveryPortTag);

ProjectPlugins/GethPlugin/GethNode.cs

@@ -1,8 +1,11 @@
 using Core;
 using KubernetesWorkflow.Types;
 using Logging;
+using Nethereum.ABI.FunctionEncoding.Attributes;
 using Nethereum.Contracts;
+using Nethereum.RPC.Eth.DTOs;
 using NethereumWorkflow;
+using Utils;
 
 namespace GethPlugin
 {
@@ -13,19 +16,23 @@ namespace GethPlugin
         Ether GetEthBalance();
         Ether GetEthBalance(IHasEthAddress address);
         Ether GetEthBalance(EthAddress address);
-        void SendEth(IHasEthAddress account, Ether eth);
-        void SendEth(EthAddress account, Ether eth);
+        string SendEth(IHasEthAddress account, Ether eth);
+        string SendEth(EthAddress account, Ether eth);
         TResult Call<TFunction, TResult>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new();
-        void SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new();
+        string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new();
+        Transaction GetTransaction(string transactionHash);
         decimal? GetSyncedBlockNumber();
         bool IsContractAvailable(string abi, string contractAddress);
+        GethBootstrapNode GetBootstrapRecord();
+        List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new();
+        List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new();
     }
 
-    public class GethNode : IGethNode
+    public class DeploymentGethNode : BaseGethNode, IGethNode
     {
         private readonly ILog log;
 
-        public GethNode(ILog log, GethDeployment startResult)
+        public DeploymentGethNode(ILog log, GethDeployment startResult)
         {
             this.log = log;
             StartResult = startResult;
@@ -34,6 +41,59 @@ namespace GethPlugin
         public GethDeployment StartResult { get; }
         public RunningContainer Container => StartResult.Container;
 
+        public GethBootstrapNode GetBootstrapRecord()
+        {
+            var address = StartResult.Container.GetInternalAddress(GethContainerRecipe.ListenPortTag);
+
+            return new GethBootstrapNode(
+                publicKey: StartResult.PubKey,
+                ipAddress: address.Host.Replace("http://", ""),
+                port: address.Port
+            );
+        }
+
+        protected override NethereumInteraction StartInteraction()
+        {
+            var address = StartResult.Container.GetAddress(log, GethContainerRecipe.HttpPortTag);
+            var account = StartResult.Account;
+
+            var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);
+            return creator.CreateWorkflow();
+        }
+    }
+
+    public class CustomGethNode : BaseGethNode, IGethNode
+    {
+        private readonly ILog log;
+        private readonly string gethHost;
+        private readonly int gethPort;
+        private readonly string privateKey;
+
+        public GethDeployment StartResult => throw new NotImplementedException();
+        public RunningContainer Container => throw new NotImplementedException();
+
+        public CustomGethNode(ILog log, string gethHost, int gethPort, string privateKey)
+        {
+            this.log = log;
+            this.gethHost = gethHost;
+            this.gethPort = gethPort;
+            this.privateKey = privateKey;
+        }
+
+        public GethBootstrapNode GetBootstrapRecord()
+        {
+            throw new NotImplementedException();
+        }
+
+        protected override NethereumInteraction StartInteraction()
+        {
+            var creator = new NethereumInteractionCreator(log, gethHost, gethPort, privateKey);
+            return creator.CreateWorkflow();
+        }
+    }
+
+    public abstract class BaseGethNode
+    {
         public Ether GetEthBalance()
         {
             return StartInteraction().GetEthBalance().Eth();
@@ -49,14 +109,14 @@ namespace GethPlugin
             return StartInteraction().GetEthBalance(address.Address).Eth();
         }
 
-        public void SendEth(IHasEthAddress owner, Ether eth)
+        public string SendEth(IHasEthAddress owner, Ether eth)
         {
-            SendEth(owner.EthAddress, eth);
+            return SendEth(owner.EthAddress, eth);
         }
 
-        public void SendEth(EthAddress account, Ether eth)
+        public string SendEth(EthAddress account, Ether eth)
         {
-            StartInteraction().SendEth(account.Address, eth.Eth);
+            return StartInteraction().SendEth(account.Address, eth.Eth);
         }
 
         public TResult Call<TFunction, TResult>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
@@ -64,18 +124,14 @@ namespace GethPlugin
             return StartInteraction().Call<TFunction, TResult>(contractAddress, function);
         }
 
-        public void SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
+        public string SendTransaction<TFunction>(string contractAddress, TFunction function) where TFunction : FunctionMessage, new()
         {
-            StartInteraction().SendTransaction(contractAddress, function);
+            return StartInteraction().SendTransaction(contractAddress, function);
         }
 
-        private NethereumInteraction StartInteraction()
+        public Transaction GetTransaction(string transactionHash)
         {
-            var address = StartResult.Container.GetAddress(log, GethContainerRecipe.HttpPortTag);
-            var account = StartResult.Account;
-
-            var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);
-            return creator.CreateWorkflow();
+            return StartInteraction().GetTransaction(transactionHash);
         }
 
         public decimal? GetSyncedBlockNumber()
@@ -87,5 +143,17 @@ namespace GethPlugin
         {
             return StartInteraction().IsContractAvailable(abi, contractAddress);
         }
+
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new()
+        {
+            return StartInteraction().GetEvents<TEvent>(address, fromBlockNumber, toBlockNumber);
+        }
+
+        public List<EventLog<TEvent>> GetEvents<TEvent>(string address, TimeRange timeRange) where TEvent : IEventDTO, new()
+        {
+            return StartInteraction().GetEvents<TEvent>(address, timeRange);
+        }
+
+        protected abstract NethereumInteraction StartInteraction();
     }
 }

ProjectPlugins/GethPlugin/GethStarter.cs

@@ -44,7 +44,7 @@ namespace GethPlugin
         public IGethNode WrapGethContainer(GethDeployment startResult)
         {
             startResult = SerializeGate.Gate(startResult);
-            return new GethNode(tools.GetLog(), startResult);
+            return new DeploymentGethNode(tools.GetLog(), startResult);
         }
 
         private void Log(string msg)

ProjectPlugins/GethPlugin/GethStartupConfig.cs

@@ -3,6 +3,7 @@
     public interface IGethSetup
     {
         IGethSetup IsMiner();
+        IGethSetup WithBootstrapNode(IGethNode node);
         IGethSetup WithBootstrapNode(GethBootstrapNode node);
         IGethSetup WithName(string name);
         IGethSetup AsPublicTestNet(GethTestNetConfig gethTestNetConfig);
@@ -15,6 +16,11 @@
         public string? NameOverride { get; private set; }
         public GethTestNetConfig? IsPublicTestNet { get; private set; }
 
+        public IGethSetup WithBootstrapNode(IGethNode node)
+        {
+            return WithBootstrapNode(node.GetBootstrapRecord());
+        }
+
         public IGethSetup WithBootstrapNode(GethBootstrapNode node)
         {
             BootstrapNode = node;
@@ -42,14 +48,12 @@
 
     public class GethTestNetConfig
     {
-        public GethTestNetConfig(string publicIp, int discoveryPort, int listenPort)
+        public GethTestNetConfig(int discoveryPort, int listenPort)
         {
-            PublicIp = publicIp;
             DiscoveryPort = discoveryPort;
             ListenPort = listenPort;
         }
 
-        public string PublicIp { get; }
         public int DiscoveryPort { get; }
         public int ListenPort { get; }
     }

ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs

@@ -14,6 +14,8 @@ namespace MetricsPlugin
         {
             var config = startupConfig.Get<PrometheusStartupConfig>();
 
+            SetSchedulingAffinity(notIn: "false");
+
             AddExposedPortAndVar("PROM_PORT", PortTag);
             AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
         }

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -18,6 +18,8 @@ namespace MetricsPlugin
 
         public RunningContainers CollectMetricsFor(IMetricsScrapeTarget[] targets)
         {
+            if (!targets.Any()) throw new ArgumentException(nameof(targets) + " must not be empty.");
+
             Log($"Starting metrics server for {targets.Length} targets...");
             var startupConfig = new StartupConfig();
             startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(targets)));

Tests/CodexContinuousTests/Configuration.cs

@@ -24,8 +24,8 @@ namespace ContinuousTests
         [Uniform("stop", "s", "STOPONFAIL", false, "If greater than zero, runner will stop after this many test failures and download all cluster container logs. 0 by default.")]
         public int StopOnFailure { get; set; } = 0;
 
-        [Uniform("target-duration", "td", "TARGETDURATION", false, "If greater than zero, runner will run for this many seconds before stopping.")]
-        public int TargetDurationSeconds { get; set; } = 0;
+        [Uniform("target-duration", "td", "TARGETDURATION", false, "If set, runner will run for this length of time before stopping. Supports seconds, or '1d2h3m4s' format.")]
+        public string TargetDurationSeconds { get; set; } = string.Empty;
 
         [Uniform("filter", "f", "FILTER", false, "If set, runs only tests whose names contain any of the filter strings. Comma-separated. Case sensitive.")]
         public string Filter { get; set; } = string.Empty;

Tests/CodexContinuousTests/ContinuousTestRunner.cs

@@ -1,7 +1,9 @@
-using DistTestCore.Logs;
+using DistTestCore;
+using DistTestCore.Logs;
 using Logging;
 using Newtonsoft.Json;
 using Utils;
+using TaskFactory = Utils.TaskFactory;
 
 namespace ContinuousTests
 {
@@ -25,10 +27,11 @@ namespace ContinuousTests
             var startTime = DateTime.UtcNow;
 
             var overviewLog = new LogSplitter(
-                new FixtureLog(logConfig, startTime, "Overview"),
+                new FixtureLog(logConfig, startTime, config.CodexDeployment.Id, "Overview"),
                 new ConsoleLog()
             );
-            var statusLog = new StatusLog(logConfig, startTime, "continuous-tests", "ContinuousTestRun");
+            var statusLog = new StatusLog(logConfig, startTime, "continuous-tests", config.CodexDeployment.Id,
+                "ContinuousTestRun");
 
             overviewLog.Log("Initializing...");
 
@@ -43,6 +46,7 @@ namespace ContinuousTests
             var taskFactory = new TaskFactory();
             overviewLog.Log("Startup checks passed. Configuration:");
             overviewLog.Log(JsonConvert.SerializeObject(config, Formatting.Indented));
+            overviewLog.Log("Test framework revision: " + GitInfo.GetStatus());
             overviewLog.Log("Continuous tests starting...");
             overviewLog.Log("");
             var allTests = testFactory.CreateTests();
@@ -57,7 +61,7 @@ namespace ContinuousTests
             }
             else
             {
-                var testLoops = filteredTests.Select(t => new TestLoop(entryPointFactory, taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
+                var testLoops = filteredTests.Select(t => new TestLoop(entryPointFactory, taskFactory, config, overviewLog, statusLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
 
                 foreach (var testLoop in testLoops)
                 {
@@ -96,19 +100,20 @@ namespace ContinuousTests
 
         private void WaitUntilFinished(LogSplitter overviewLog, StatusLog statusLog, DateTime startTime, TestLoop[] testLoops)
         {
-            var testDuration = Time.FormatDuration(DateTime.UtcNow - startTime);
+            var testDuration = (DateTime.UtcNow - startTime).TotalSeconds.ToString();
             var testData = FormatTestRuns(testLoops);
             overviewLog.Log("Total duration: " + testDuration);
 
-            if (config.TargetDurationSeconds > 0)
+            if (!string.IsNullOrEmpty(config.TargetDurationSeconds))
             {
-                var targetDuration = TimeSpan.FromSeconds(config.TargetDurationSeconds);
+                var targetDuration = Time.ParseTimespan(config.TargetDurationSeconds);
                 var wasCancelled = cancelToken.WaitHandle.WaitOne(targetDuration);
                 if (!wasCancelled)
                 {
                     Cancellation.Cts.Cancel();
                     overviewLog.Log($"Congratulations! The targer duration has been reached! ({Time.FormatDuration(targetDuration)})");
                     statusLog.ConcludeTest("Passed", testDuration, testData);
+                    Environment.ExitCode = 0;
                     return;
                 }
             }
@@ -117,6 +122,7 @@ namespace ContinuousTests
                 cancelToken.WaitHandle.WaitOne();
             }
             statusLog.ConcludeTest("Failed", testDuration, testData);
+            Environment.ExitCode = 1;
         }
 
         private Dictionary<string, string> FormatTestRuns(TestLoop[] testLoops)
@@ -124,7 +130,8 @@ namespace ContinuousTests
             var result = new Dictionary<string, string>();
             foreach (var testLoop in testLoops)
             {
-                result.Add($"ctest-{testLoop.Name}", $"passes: {testLoop.NumberOfPasses} - failures: {testLoop.NumberOfFailures}");
+                result.Add("testname", testLoop.Name);
+                result.Add($"summary", $"passes: {testLoop.NumberOfPasses} - failures: {testLoop.NumberOfFailures}");
             }
             return result;
         }

Tests/CodexContinuousTests/Program.cs

@@ -15,8 +15,15 @@ public class Program
 
             Cancellation.Cts.Cancel();
         };
-        
-        runner.Run();
+
+        try
+        {
+            runner.Run();
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine(ex.ToString());
+        }
         Console.WriteLine("Done.");
     }
 }

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -6,6 +6,7 @@ using CodexPlugin;
 using DistTestCore.Logs;
 using Core;
 using KubernetesWorkflow.Types;
+using TaskFactory = Utils.TaskFactory;
 
 namespace ContinuousTests
 {
@@ -16,6 +17,7 @@ namespace ContinuousTests
         private readonly TaskFactory taskFactory;
         private readonly Configuration config;
         private readonly ILog overviewLog;
+        private readonly StatusLog statusLog;
         private readonly TestHandle handle;
         private readonly CancellationToken cancelToken;
         private readonly ICodexNode[] nodes;
@@ -23,16 +25,20 @@ namespace ContinuousTests
         private readonly string testName;
         private static int failureCount = 0;
 
-        public SingleTestRun(EntryPointFactory entryPointFactory, TaskFactory taskFactory, Configuration config, ILog overviewLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken)
+        public SingleTestRun(EntryPointFactory entryPointFactory,
+            TaskFactory taskFactory, Configuration config, ILog overviewLog, StatusLog statusLog, TestHandle handle,
+            StartupChecker startupChecker, CancellationToken cancelToken, string deployId)
         {
             this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
+            this.statusLog = statusLog;
             this.handle = handle;
             this.cancelToken = cancelToken;
             testName = handle.Test.GetType().Name;
-            fixtureLog = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, testName);
-            entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, fixtureLog);
+            fixtureLog = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, deployId, testName);
+            entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath,
+                config.CodexDeployment.Metadata.KubeNamespace, fixtureLog);
             ApplyLogReplacements(fixtureLog, startupChecker);
 
             nodes = CreateRandomNodes();
@@ -57,30 +63,34 @@ namespace ContinuousTests
                     overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex);
                     Environment.Exit(-1);
                 }
-            });
+            }, nameof(SingleTestRun));
         }
 
         private void RunTest(Action<bool> resultHandler)
         {
             var testStart = DateTime.UtcNow;
-            
+            TimeSpan duration = TimeSpan.Zero;
+
             try
             {
                 RunTestMoments();
+                duration = DateTime.UtcNow - testStart;
 
-                var duration = DateTime.UtcNow - testStart;
                 OverviewLog($" > Test passed. ({Time.FormatDuration(duration)})");
+                UpdateStatusLogPassed(testStart, duration);
 
                 if (!config.KeepPassedTestLogs)
                 {
                     fixtureLog.Delete();
                 }
+
                 resultHandler(true);
             }
             catch (Exception ex)
             {
                 fixtureLog.Error("Test run failed with exception: " + ex);
                 fixtureLog.MarkAsFailed();
+                UpdateStatusLogFailed(testStart, duration, ex.ToString());
 
                 DownloadContainerLogs(testStart);
 
@@ -108,6 +118,7 @@ namespace ContinuousTests
             {
                 effectiveStart = config.CodexDeployment.Metadata.StartUtc.Subtract(TimeSpan.FromSeconds(30));
             }
+
             var effectiveEnd = DateTime.UtcNow;
             var elasticSearchLogDownloader = new ElasticSearchLogDownloader(entryPoint.Tools, fixtureLog);
 
@@ -116,14 +127,17 @@ namespace ContinuousTests
                 var container = node.Container;
                 var deploymentName = container.RunningContainers.StartResult.Deployment.Name;
                 var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
-                var openingLine = $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().id}";
-                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart, effectiveEnd, openingLine);
+                var openingLine =
+                    $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().id}";
+                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart,
+                    effectiveEnd, openingLine);
             }
         }
 
         private void ApplyLogReplacements(FixtureLog fixtureLog, StartupChecker startupChecker)
         {
-            foreach (var replacement in startupChecker.LogReplacements) fixtureLog.AddStringReplace(replacement.From, replacement.To);
+            foreach (var replacement in startupChecker.LogReplacements)
+                fixtureLog.AddStringReplace(replacement.From, replacement.To);
         }
 
         private void RunTestMoments()
@@ -155,9 +169,11 @@ namespace ContinuousTests
                     {
                         ThrowFailTest();
                     }
+
                     return;
                 }
             }
+
             fixtureLog.Log("Test run has been cancelled.");
         }
 
@@ -170,6 +186,37 @@ namespace ContinuousTests
             throw new Exception(exceptionsMessage);
         }
 
+        private void UpdateStatusLogFailed(DateTime testStart, TimeSpan duration, string error)
+        {
+            statusLog.ConcludeTest("Failed", duration, CreateStatusLogData(testStart, error));
+        }
+
+        private void UpdateStatusLogPassed(DateTime testStart, TimeSpan duration)
+        {
+            statusLog.ConcludeTest("Passed", duration, CreateStatusLogData(testStart, "OK"));
+        }
+
+        private Dictionary<string, string> CreateStatusLogData(DateTime testStart, string message)
+        {
+            var result = entryPoint.GetPluginMetadata();
+            result.Add("teststart", testStart.ToString("o"));
+            result.Add("testname", testName);
+            result.Add("message", message);
+            result.Add("involvedpods", string.Join(",", nodes.Select(n => n.GetName())));
+            result.Add("involvedpodnames", string.Join(",", nodes.Select(n => n.GetPodInfo().Name)));
+
+            var error = message.Split(Environment.NewLine).First();
+            if (error.Contains(":")) error = error.Substring(1 + error.LastIndexOf(":"));
+            result.Add("error", error);
+
+            var upload = nodes.Select(n => n.TransferSpeeds.GetUploadSpeed()).ToList()!.OptionalAverage();
+            var download = nodes.Select(n => n.TransferSpeeds.GetDownloadSpeed()).ToList()!.OptionalAverage();
+            if (upload != null) result.Add("avgupload", upload.ToString());
+            if (download != null) result.Add("avgdownload", download.ToString());
+
+            return result;
+        }
+
         private string GetCombinedExceptionsMessage(Exception[] exceptions)
         {
             return string.Join(Environment.NewLine, exceptions.Select(ex => ex.ToString()));
@@ -186,6 +233,7 @@ namespace ContinuousTests
             {
                 return UnpackException(a.InnerExceptions.First());
             }
+
             if (exception is TargetInvocationException t)
             {
                 return UnpackException(t.InnerException!);
@@ -258,7 +306,8 @@ namespace ContinuousTests
             {
                 result[i] = containers.PickOneRandom();
             }
+
             return result;
         }
     }
-}
+}

Tests/CodexContinuousTests/StartupChecker.cs

@@ -23,7 +23,8 @@ namespace ContinuousTests
 
         public void Check()
         {
-            var log = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, "StartupChecks");
+            var log = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, config.CodexDeployment.Id,
+                "StartupChecks");
             log.Log("Starting continuous test run...");
             IncludeDeploymentConfiguration(log);
             log.Log("Checking configuration...");
@@ -46,13 +47,15 @@ namespace ContinuousTests
                 {
                     var podInfo = workflow.GetPodInfo(container);
                     log.Log($"Codex environment variables for '{container.Name}':");
-                    log.Log($"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
+                    log.Log(
+                        $"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
                         $"Pod name: {podInfo.Name} - Deployment name: {instance.Containers.StartResult.Deployment.Name}");
                     var codexVars = container.Recipe.EnvVars;
                     foreach (var vars in codexVars) log.Log(vars.ToString());
                     log.Log("");
                 }
             }
+
             log.Log($"Deployment metadata: {JsonConvert.SerializeObject(deployment.Metadata)}");
             log.Log("");
         }
@@ -64,6 +67,7 @@ namespace ContinuousTests
             {
                 throw new Exception("Unable to find any tests.");
             }
+
             foreach (var test in tests)
             {
                 cancelToken.ThrowIfCancellationRequested();
@@ -87,7 +91,8 @@ namespace ContinuousTests
 
         private void CheckCodexNodes(BaseLog log, Configuration config)
         {
-            var nodes = entryPoint.CreateInterface().WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray());
+            var nodes = entryPoint.CreateInterface()
+                .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray());
             var pass = true;
             foreach (var n in nodes)
             {
@@ -106,6 +111,7 @@ namespace ContinuousTests
                     pass = false;
                 }
             }
+
             if (!pass)
             {
                 throw new Exception("Not all codex nodes responded.");
@@ -126,6 +132,7 @@ namespace ContinuousTests
             {
                 return false;
             }
+
             return true;
         }
 
@@ -145,7 +152,8 @@ namespace ContinuousTests
                 propertyName: nameof(ContinuousTest.CustomK8sNamespace));
         }
 
-        private void DuplicatesCheck(ContinuousTest[] tests, List<string> errors, Func<ContinuousTest, bool> considerCondition, Func<ContinuousTest, object> getValue, string propertyName)
+        private void DuplicatesCheck(ContinuousTest[] tests, List<string> errors,
+            Func<ContinuousTest, bool> considerCondition, Func<ContinuousTest, object> getValue, string propertyName)
         {
             foreach (var test in tests)
             {
@@ -155,7 +163,8 @@ namespace ContinuousTests
                     if (duplicates.Any())
                     {
                         duplicates.Add(test);
-                        errors.Add($"Tests '{string.Join(",", duplicates.Select(d => d.Name))}' have the same '{propertyName}'. These must be unique.");
+                        errors.Add(
+                            $"Tests '{string.Join(",", duplicates.Select(d => d.Name))}' have the same '{propertyName}'. These must be unique.");
                         return;
                     }
                 }
@@ -170,14 +179,16 @@ namespace ContinuousTests
                 {
                     if (test.RequiredNumberOfNodes < 1)
                     {
-                        errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Test must require > 0 nodes, or -1 to select all nodes.");
+                        errors.Add(
+                            $"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Test must require > 0 nodes, or -1 to select all nodes.");
                     }
                     else if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexInstances.Length)
                     {
-                        errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexInstances.Length}");
+                        errors.Add(
+                            $"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexInstances.Length}");
                     }
                 }
             }
         }
     }
-}
+}

Tests/CodexContinuousTests/TestLoop.cs

@@ -1,4 +1,6 @@
-using Logging;
+using DistTestCore.Logs;
+using Logging;
+using TaskFactory = Utils.TaskFactory;
 
 namespace ContinuousTests
 {
@@ -8,6 +10,7 @@ namespace ContinuousTests
         private readonly TaskFactory taskFactory;
         private readonly Configuration config;
         private readonly ILog overviewLog;
+        private readonly StatusLog statusLog;
         private readonly Type testType;
         private readonly TimeSpan runsEvery;
         private readonly StartupChecker startupChecker;
@@ -15,12 +18,13 @@ namespace ContinuousTests
         private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset);
         private static object testLock = new object();
 
-        public TestLoop(EntryPointFactory entryPointFactory, TaskFactory taskFactory, Configuration config, ILog overviewLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken)
+        public TestLoop(EntryPointFactory entryPointFactory, TaskFactory taskFactory, Configuration config, ILog overviewLog, StatusLog statusLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken)
         {
             this.entryPointFactory = entryPointFactory;
             this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
+            this.statusLog = statusLog;
             this.testType = testType;
             this.runsEvery = runsEvery;
             this.startupChecker = startupChecker;
@@ -66,14 +70,15 @@ namespace ContinuousTests
                     overviewLog.Error("Test infra failure: TestLoop failed with " + ex);
                     Environment.Exit(-1);
                 }
-            });
+            }, nameof(TestLoop));
         }
 
         private void StartTest()
         {
             var test = (ContinuousTest)Activator.CreateInstance(testType)!;
             var handle = new TestHandle(test);
-            var run = new SingleTestRun(entryPointFactory, taskFactory, config, overviewLog, handle, startupChecker, cancelToken);
+            var run = new SingleTestRun(entryPointFactory, taskFactory, config, overviewLog, statusLog, handle,
+                startupChecker, cancelToken, config.CodexDeployment.Id);
 
             runFinishedHandle.Reset();
             run.Run(runFinishedHandle, result =>

Tests/CodexContinuousTests/reports/CodexTestNetReport-November2023.md

@@ -0,0 +1,41 @@
+# Codex Continuous Test-net Report
+Date: 05-12-2023
+
+Report for: 11-2023
+
+
+## Continuous test-net Status
+Continuous test runs (which can take many hours or days) can easily be started by team member from the github actions UI. Results are collected and displayed in Grafana. For the time being, we're suspending the effort to have a network of Codex nodes "always online" and continuously being tested, until overal reliability improves.
+
+## Deployment Configuration
+Continous Test-net is deployed to the kubernetes cluster with the following configuration:
+
+5x Codex Nodes:
+- Log-level: Trace
+- Storage quota: 20480 MB
+- Storage sell: 1024 MB
+- Min price: 1024
+- Max collateral: 1024
+- Max duration: 3600000 seconds
+- Block-TTL*: 99999999 seconds
+- Block-MI*: 99999999 seconds
+- Block-MN*: 100 blocks
+3 of these 5 nodes have:
+- Validator: true
+
+## Test Overview
+| Changes | Test             | Description                    | Status  | Results              |
+|---------|------------------|--------------------------------|---------|----------------------|
+| todo    | Two-client test  | See report for July 2023.      | Faulted | Test reliably fails. |
+| todo    | Two-client test* | See report for September 2023. | Faulted | Test reliably fails. |
+| todo    | HoldMyBeer test  | See report for August 2023.    | todo    | todo                 |
+| todo    | Peers test       | See report for August 2023.    | todo    | todo                 |
+
+## Resulting changes
+As a result of the testing efforts in 11-2023, these changes were made:
+1. todo
+
+## Action Points
+- Debugging efforts continuou
+- 
+

Tests/CodexContinuousTests/reports/CodexTestNetReport-October2023.md

@@ -0,0 +1,52 @@
+# Codex Continuous Test-net Report
+Date: 13-11-2023
+
+Report for: 10-2023
+
+
+## Test-net Status
+- Start of month: Offline - stopped
+- End of month: Offline - stopped
+
+(Stopped: The number of tests that can successfully run on the test-net is not high enough to justify the cost of leaving it running.)
+
+## Deployment Configuration
+Continous Test-net is deployed to the kubernetes cluster with the following configuration:
+
+5x Codex Nodes:
+- Log-level: Trace
+- Storage quota: 2048 MB
+- Storage sell: 1024 MB
+- Min price: 1024
+- Max collateral: 1024
+- Max duration: 3600000 seconds
+- Block-TTL*: 180 seconds
+- Block-MI*: 120 seconds
+- Block-MN*: 10000 blocks
+
+3 of these 5 nodes have:
+- Validator: true
+
+Kubernetes namespace: 'codex-continuous-tests'
+* Some tests have been performed with alternative (disabled) maintenance parameters:
+- Block-TTL: 99999999 seconds
+- Block-MI: 99999999 seconds
+- Block-MN: 100 blocks
+
+## Test Overview
+| Changes             | Test             | Description                    | Status     | Results                                                       |
+|---------------------|------------------|--------------------------------|------------|---------------------------------------------------------------|
+| No change           | Two-client test  | See report for July 2023.      | Faulted    | Test reliably fails. Both upload and download failures occur. |
+| No change           | Two-client test* | See report for September 2023. | Faulted    | Test reliably fails. Both upload and download failures occur. |
+| Possible regression | HoldMyBeer test  | See report for August 2023.    | Unreliable | Successful runs of 48h have not been observed in October.     |
+| Possible regression | Peers test       | See report for August 2023.    | Unreliable | Successful runs of 48h have not been observed in October.     |
+
+## Resulting changes
+As a result of the testing efforts in 10-2023, these changes were made:
+1. Consolidation of test logs and metrics using grafana and elastic-search.
+1. Investment made in profiling instrumentation in Codex codebase.
+1. Some testing effort has been diverted to preparing the necessary infrastructure for the creation of a public testnet by 1-December-2023.
+
+## Action Points
+- Debugging efforts continuou
+- Some effort remains allocated to deploying and supporting the public testnet

Tests/CodexTests/AutoBootstrapDistTest.cs

@@ -23,5 +23,6 @@ namespace CodexTests
         }
 
         protected ICodexNode? BootstrapNode { get; private set; }
+        
     }
 }

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -3,6 +3,7 @@ using CodexPlugin;
 using DistTestCore;
 using GethPlugin;
 using MetricsPlugin;
+using Nethereum.Hex.HexConvertors.Extensions;
 using NUnit.Framework;
 using Utils;
 
@@ -59,6 +60,7 @@ namespace CodexTests.BasicTests
             var contracts = Ci.StartCodexContracts(geth);
 
             var seller = AddCodex(s => s
+                .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn))
                 .WithStorageQuota(11.GB())
                 .EnableMarketplace(geth, contracts, initialEth: 10.Eth(), initialTokens: sellerInitialBalance, isValidator: true)
                 .WithSimulateProofFailures(failEveryNProofs: 3));
@@ -88,12 +90,57 @@ namespace CodexTests.BasicTests
 
             purchaseContract.WaitForStorageContractStarted(fileSize);
 
+            var requests = contracts.GetStorageRequests(GetTestRunTimeRange());
+            Assert.That(requests.Length, Is.EqualTo(1));
+            var request = requests.Single();
+            Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Started));
+            Assert.That(request.ClientAddress, Is.EqualTo(buyer.EthAddress));
+            Assert.That(request.Ask.Slots, Is.EqualTo(1));
+
             AssertBalance(contracts, seller, Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
 
+            var requestFulfilledEvents = contracts.GetRequestFulfilledEvents(GetTestRunTimeRange());
+            Assert.That(requestFulfilledEvents.Length, Is.EqualTo(1));
+            CollectionAssert.AreEqual(request.RequestId, requestFulfilledEvents[0].RequestId);
+            var filledSlotEvents = contracts.GetSlotFilledEvents(GetTestRunTimeRange());
+            Assert.That(filledSlotEvents.Length, Is.EqualTo(1));
+            var filledSlotEvent = filledSlotEvents.Single();
+            Assert.That(filledSlotEvent.SlotIndex.IsZero);
+            Assert.That(filledSlotEvent.RequestId.ToHex(), Is.EqualTo(request.RequestId.ToHex()));
+            Assert.That(filledSlotEvent.Host, Is.EqualTo(seller.EthAddress));
+
+            var slotHost = contracts.GetSlotHost(request, 0);
+            Assert.That(slotHost, Is.EqualTo(seller.EthAddress));
+
             purchaseContract.WaitForStorageContractFinished();
 
             AssertBalance(contracts, seller, Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
             AssertBalance(contracts, buyer, Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");
+            Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished));
+
+            var log = Ci.DownloadLog(seller);
+            log.AssertLogContains("Received a request to store a slot!");
+            log.AssertLogContains("Received proof challenge");
+            log.AssertLogContains("Collecting input for proof");
+
+            //CheckLogForErrors(seller, buyer);
+        }
+
+        [Test]
+        public void GethBootstrapTest()
+        {
+            var boot = Ci.StartGethNode(s => s.WithName("boot").IsMiner());
+            var disconnected = Ci.StartGethNode(s => s.WithName("disconnected"));
+            var follow = Ci.StartGethNode(s => s.WithBootstrapNode(boot).WithName("follow"));
+
+            Thread.Sleep(12000);
+
+            var bootN = boot.GetSyncedBlockNumber();
+            var discN = disconnected.GetSyncedBlockNumber();
+            var followN = follow.GetSyncedBlockNumber();
+
+            Assert.That(bootN, Is.EqualTo(followN));
+            Assert.That(discN, Is.LessThan(bootN));
         }
     }
 }

Tests/CodexTests/BasicTests/OneClientTests.cs

@@ -6,7 +6,7 @@ using Utils;
 namespace CodexTests.BasicTests
 {
     [TestFixture]
-    public class OneClientTests : DistTest
+    public class OneClientTests : CodexDistTest
     {
         [Test]
         public void OneClientTest()

Tests/CodexTests/BasicTests/TwoClientTests.cs

@@ -1,12 +1,11 @@
 using CodexPlugin;
-using DistTestCore;
 using NUnit.Framework;
 using Utils;
 
 namespace CodexTests.BasicTests
 {
     [TestFixture]
-    public class TwoClientTests : DistTest
+    public class TwoClientTests : CodexDistTest
     {
         [Test]
         public void TwoClientTest()
@@ -51,6 +50,7 @@ namespace CodexTests.BasicTests
             var downloadedFile = secondary.DownloadContent(contentId);
 
             testFile.AssertIsEqual(downloadedFile);
+            CheckLogForErrors(primary, secondary);
         }
     }
 }

Tests/CodexTests/CodexDistTest.cs

@@ -6,14 +6,13 @@ using Core;
 using DistTestCore;
 using DistTestCore.Helpers;
 using DistTestCore.Logs;
-using NUnit.Framework;
 using NUnit.Framework.Constraints;
 
 namespace CodexTests
 {
     public class CodexDistTest : DistTest
     {
-        private readonly List<ICodexNode> onlineCodexNodes = new List<ICodexNode>();
+        private readonly Dictionary<TestLifecycle, List<ICodexNode>> onlineCodexNodes = new Dictionary<TestLifecycle, List<ICodexNode>>();
 
         public CodexDistTest()
         {
@@ -23,12 +22,6 @@ namespace CodexTests
             ProjectPlugin.Load<MetricsPlugin.MetricsPlugin>();
         }
 
-        [TearDown]
-        public void TearDownCodexFixture()
-        {
-            onlineCodexNodes.Clear();
-        }
-
         protected override void Initialize(FixtureLog fixtureLog)
         {
             var localBuilder = new LocalCodexBuilder(fixtureLog);
@@ -36,6 +29,16 @@ namespace CodexTests
             localBuilder.Build();
         }
 
+        protected override void LifecycleStart(TestLifecycle lifecycle)
+        {
+            onlineCodexNodes.Add(lifecycle, new List<ICodexNode>());
+        }
+
+        protected override void LifecycleStop(TestLifecycle lifecycle)
+        {
+            onlineCodexNodes.Remove(lifecycle);
+        }
+
         public ICodexNode AddCodex()
         {
             return AddCodex(s => { });
@@ -58,7 +61,7 @@ namespace CodexTests
                 setup(s);
                 OnCodexSetup(s);
             });
-            onlineCodexNodes.AddRange(group);
+            onlineCodexNodes[Get()].AddRange(group);
             return group;
         }
 
@@ -74,7 +77,7 @@ namespace CodexTests
 
         public IEnumerable<ICodexNode> GetAllOnlineCodexNodes()
         {
-            return onlineCodexNodes;
+            return onlineCodexNodes[Get()];
         }
 
         public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "")
@@ -82,8 +85,30 @@ namespace CodexTests
             AssertHelpers.RetryAssert(constraint, () => contracts.GetTestTokenBalance(codexNode), nameof(AssertBalance) + msg);
         }
 
+        public void CheckLogForErrors(params ICodexNode[] nodes)
+        {
+            foreach (var node in nodes) CheckLogForErrors(node);
+        }
+
+        public void CheckLogForErrors(ICodexNode node)
+        {
+            var log = Ci.DownloadLog(node);
+
+            log.AssertLogDoesNotContain("Block validation failed");
+            log.AssertLogDoesNotContain("ERR ");
+        }
+
         protected virtual void OnCodexSetup(ICodexSetup setup)
         {
         }
+
+        protected override void CollectStatusLogData(TestLifecycle lifecycle, Dictionary<string, string> data)
+        {
+            var nodes = onlineCodexNodes[lifecycle];
+            var upload = nodes.Select(n => n.TransferSpeeds.GetUploadSpeed()).ToList()!.OptionalAverage();
+            var download = nodes.Select(n => n.TransferSpeeds.GetDownloadSpeed()).ToList()!.OptionalAverage();
+            if (upload != null) data.Add("avgupload", upload.ToString());
+            if (download != null) data.Add("avgdownload", download.ToString());
+        }
     }
 }

Tests/CodexTests/Helpers/PeerDownloadTestHelpers.cs

@@ -9,7 +9,6 @@ namespace CodexTests.Helpers
     public class PeerDownloadTestHelpers : IFullConnectivityImplementation
     {
         private readonly FullConnectivityHelper helper;
-        private readonly ILog log;
         private readonly IFileManager fileManager;
         private ByteSize testFileSize;
 
@@ -17,7 +16,6 @@ namespace CodexTests.Helpers
         {
             helper = new FullConnectivityHelper(log, this);
             testFileSize = 1.MB();
-            this.log = log;
             this.fileManager = fileManager;
         }
 
@@ -45,11 +43,11 @@ namespace CodexTests.Helpers
         private PeerConnectionState CheckConnectivity(Entry from, Entry to)
         {
             var expectedFile = GenerateTestFile(from.Node, to.Node);
-            var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(expectedFile));
+            var contentId = from.Node.UploadFile(expectedFile);
 
             try
             {
-                var downloadedFile = Stopwatch.Measure(log, "Download", () => DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded"));
+                var downloadedFile = DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded");
                 expectedFile.AssertIsEqual(downloadedFile);
                 return PeerConnectionState.Connection;
             }

Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin;
+using CodexPlugin;
 using GethPlugin;
 using NUnit.Framework;
 
@@ -48,7 +49,45 @@ namespace CodexTests.PeerDiscoveryTests
 
         private void AssertAllNodesConnected()
         {
-            CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
+            var allNodes = GetAllOnlineCodexNodes();
+            CreatePeerConnectionTestHelpers().AssertFullyConnected(allNodes);
+            CheckRoutingTable(allNodes);
+        }
+
+        private void CheckRoutingTable(IEnumerable<ICodexNode> allNodes)
+        {
+            var allResponses = allNodes.Select(n => n.GetDebugInfo()).ToArray();
+
+            var errors = new List<string>();
+            foreach (var response in allResponses)
+            {
+                var error = AreAllPresent(response, allResponses);
+                if (!string.IsNullOrEmpty(error)) errors.Add(error);
+            }
+
+            if (errors.Any())
+            {
+                Assert.Fail(string.Join(Environment.NewLine, errors));
+            }
+        }
+
+        private string AreAllPresent(CodexDebugResponse info, CodexDebugResponse[] allResponses)
+        {
+            var knownIds = info.table.nodes.Select(n => n.nodeId).ToArray();
+            var allOthers = GetAllOtherResponses(info, allResponses);
+            var expectedIds = allOthers.Select(i => i.table.localNode.nodeId).ToArray();
+
+            if (!expectedIds.All(ex => knownIds.Contains(ex)))
+            {
+                return $"Node {info.id}: Not all of '{string.Join(",", expectedIds)}' were present in routing table: '{string.Join(",", knownIds)}'";
+            }
+
+            return string.Empty;
+        }
+
+        private CodexDebugResponse[] GetAllOtherResponses(CodexDebugResponse exclude, CodexDebugResponse[] allResponses)
+        {
+            return allResponses.Where(r => r.id != exclude.id).ToArray();
         }
     }
 }

Tests/CodexTests/PeerDiscoveryTests/RoutingTableTests.cs

@@ -1,63 +0,0 @@
-using CodexPlugin;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.PeerDiscoveryTests
-{
-    [TestFixture]
-    public class RoutingTableTests : AutoBootstrapDistTest
-    {
-        [TestCase(2)]
-        [TestCase(3)]
-        [TestCase(10)]
-        [TestCase(20)]
-        public void VariableNodes(int number)
-        {
-            AddCodex(number);
-
-            AssertRoutingTable();
-        }
-
-        private void AssertRoutingTable()
-        {
-            Time.Retry(CheckRoutingTable, 3, nameof(CheckRoutingTable));
-        }
-
-        private void CheckRoutingTable()
-        {
-            var all = GetAllOnlineCodexNodes();
-            var allResponses = all.Select(n => n.GetDebugInfo()).ToArray();
-
-            var errors = new List<string>();
-            foreach (var response in allResponses)
-            {
-                var error = AreAllPresent(response, allResponses);
-                if (!string.IsNullOrEmpty(error)) errors.Add(error);
-            }
-            
-            if (errors.Any())
-            {
-                Assert.Fail(string.Join(Environment.NewLine, errors));
-            }
-        }
-
-        private string AreAllPresent(CodexDebugResponse info, CodexDebugResponse[] allResponses)
-        {
-            var knownIds = info.table.nodes.Select(n => n.nodeId).ToArray();
-            var allOthers = GetAllOtherResponses(info, allResponses);
-            var expectedIds = allOthers.Select(i => i.table.localNode.nodeId).ToArray();
-
-            if (!expectedIds.All(ex => knownIds.Contains(ex)))
-            {
-                return $"Not all of '{string.Join(",", expectedIds)}' were present in routing table: '{string.Join(",", knownIds)}'";
-            }
-
-            return string.Empty;
-        }
-
-        private CodexDebugResponse[] GetAllOtherResponses(CodexDebugResponse exclude, CodexDebugResponse[] allResponses)
-        {
-            return allResponses.Where(r => r.id != exclude.id).ToArray();
-        }
-    }
-}

Tests/DistTestCore/DistTest.cs

@@ -20,16 +20,19 @@ namespace DistTestCore
         private readonly object lifecycleLock = new object();
         private readonly EntryPoint globalEntryPoint;
         private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
+        private readonly string deployId;
         
         public DistTest()
         {
             var assemblies = AppDomain.CurrentDomain.GetAssemblies();
             testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
+            
+            deployId = NameUtils.MakeDeployId();
 
             var logConfig = configuration.GetLogConfig();
             var startTime = DateTime.UtcNow;
-            fixtureLog = new FixtureLog(logConfig, startTime);
-            statusLog = new StatusLog(logConfig, startTime, "dist-tests");
+            fixtureLog = new FixtureLog(logConfig, startTime, deployId);
+            statusLog = new StatusLog(logConfig, startTime, "dist-tests", deployId);
 
             globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder());
 
@@ -58,6 +61,7 @@ namespace DistTestCore
                 throw;
             }
 
+            fixtureLog.Log("Test framework revision: " + GitInfo.GetStatus());
             fixtureLog.Log("Global setup cleanup successful");
         }
 
@@ -142,10 +146,27 @@ namespace DistTestCore
             Stopwatch.Measure(Get().Log, name, action);
         }
 
+        protected TimeRange GetTestRunTimeRange()
+        {
+            return new TimeRange(Get().TestStart, DateTime.UtcNow);
+        }
+
         protected virtual void Initialize(FixtureLog fixtureLog)
         {
         }
 
+        protected virtual void LifecycleStart(TestLifecycle lifecycle)
+        {
+        }
+
+        protected virtual void LifecycleStop(TestLifecycle lifecycle)
+        {
+        }
+
+        protected virtual void CollectStatusLogData(TestLifecycle lifecycle, Dictionary<string, string> data)
+        {
+        }
+
         protected TestLifecycle Get()
         {
             lock (lifecycleLock)
@@ -163,8 +184,9 @@ namespace DistTestCore
                 lock (lifecycleLock)
                 {
                     var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString();
-                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace);
+                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace, deployId);
                     lifecycles.Add(testName, lifecycle);
+                    LifecycleStart(lifecycle);
                 }
             });
         }
@@ -174,13 +196,16 @@ namespace DistTestCore
             var lifecycle = Get();
             var testResult = GetTestResult();
             var testDuration = lifecycle.GetTestDuration();
+            var data = lifecycle.GetPluginMetadata();
+            CollectStatusLogData(lifecycle, data);
             fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
-            statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetPluginMetadata());
+            statusLog.ConcludeTest(testResult, testDuration, data);
             Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
             {
                 WriteEndTestLog(lifecycle.Log);
 
                 IncludeLogsOnTestFailure(lifecycle);
+                LifecycleStop(lifecycle);
                 lifecycle.DeleteAllResources();
                 lifecycle = null!;
             });

Tests/DistTestCore/DistTestCore.csproj

@@ -8,6 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+	  <PackageReference Include="LibGit2Sharp" Version="0.28.0" />
 	  <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
 	  <PackageReference Include="nunit" Version="3.13.3" />
 	  <PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />

Tests/DistTestCore/DownloadedLogExtensions.cs

@@ -9,5 +9,18 @@ namespace DistTestCore
         {
             Assert.That(log.DoesLogContain(expectedString), $"Did not find '{expectedString}' in log.");
         }
+
+        public static void AssertLogDoesNotContain(this IDownloadedLog log, params string[] unexpectedStrings)
+        {
+            var errors = new List<string>();
+            foreach (var str in unexpectedStrings)
+            {
+                if (log.DoesLogContain(str))
+                {
+                    errors.Add($"Did find '{str}' in log.");
+                }
+            }
+            CollectionAssert.IsEmpty(errors);
+        }
     }
 }

Tests/DistTestCore/GitInfo.cs

@@ -0,0 +1,39 @@
+using KubernetesWorkflow;
+using LibGit2Sharp;
+using System.Reflection;
+
+namespace DistTestCore
+{
+    public static class GitInfo
+    {
+        private static string? status = null;
+
+        public static string GetStatus()
+        {
+            if (status == null) status = DetermineStatus();
+            return status;
+        }
+
+        private static string DetermineStatus()
+        {
+            var path = FindGitPath();
+            if (path == null) return "unknown";
+
+            using var repo = new Repository(path);
+            var sha = repo.Head.Tip.Sha.Substring(0, 7);
+
+            return K8sNameUtils.Format(sha);
+        }
+
+        private static string? FindGitPath()
+        {
+            var path = Repository.Discover(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
+            if (!string.IsNullOrEmpty(path)) return path;
+                
+            path = Repository.Discover(Directory.GetCurrentDirectory());
+            if (!string.IsNullOrEmpty(path)) return path;
+
+            return null;
+        }
+    }
+}

Tests/DistTestCore/Logs/BaseTestLog.cs

@@ -5,13 +5,18 @@ namespace DistTestCore.Logs
     public abstract class BaseTestLog : BaseLog
     {
         private bool hasFailed;
+        private readonly string deployId;
+
+        protected BaseTestLog(string deployId)
+        {
+            this.deployId = deployId;
+        }
 
         public void WriteLogTag()
         {
-            var runId = NameUtils.GetRunId();
             var category = NameUtils.GetCategoryName();
             var name = NameUtils.GetTestMethodName();
-            LogFile.WriteRaw($"{runId} {category} {name}");
+            LogFile.WriteRaw($"{deployId} {category} {name}");
         }
 
         public void MarkAsFailed()

Tests/DistTestCore/Logs/FixtureLog.cs

@@ -5,15 +5,17 @@ namespace DistTestCore.Logs
     public class FixtureLog : BaseTestLog
     {
         private readonly string fullName;
+        private readonly string deployId;
 
-        public FixtureLog(LogConfig config, DateTime start, string name = "")
+        public FixtureLog(LogConfig config, DateTime start, string deployId, string name = "") : base(deployId)
         {
+            this.deployId = deployId;
             fullName = NameUtils.GetFixtureFullName(config, start, name);
         }
 
         public TestLog CreateTestLog(string name = "")
         {
-            return new TestLog(fullName, name);
+            return new TestLog(fullName, deployId, name);
         }
 
         public void DeleteFolder()
@@ -26,4 +28,4 @@ namespace DistTestCore.Logs
             return fullName;
         }
     }
-}
+}

Tests/DistTestCore/Logs/StatusLog.cs

@@ -1,33 +1,42 @@
 using Logging;
 using Newtonsoft.Json;
+using System.Globalization;
 
 namespace DistTestCore.Logs
 {
     public class StatusLog
     {
-        private readonly object fileLock = new object();
+        private readonly object fileLock = new();
+        private readonly string deployId;
         private readonly string fullName;
         private readonly string fixtureName;
         private readonly string testType;
 
-        public StatusLog(LogConfig config, DateTime start, string testType, string name = "")
+        public StatusLog(LogConfig config, DateTime start, string testType, string deployId, string name = "")
         {
             fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log";
             fixtureName = NameUtils.GetRawFixtureName();
             this.testType = testType;
+            this.deployId = deployId;
+        }
+
+        public void ConcludeTest(string resultStatus, TimeSpan testDuration, Dictionary<string, string> data)
+        {
+            ConcludeTest(resultStatus, testDuration.TotalSeconds.ToString(CultureInfo.InvariantCulture), data);
         }
 
         public void ConcludeTest(string resultStatus, string testDuration, Dictionary<string, string> data)
         {
             data.Add("timestamp", DateTime.UtcNow.ToString("o"));
-            data.Add("runid", NameUtils.GetRunId());
+            data.Add("deployid", deployId);
             data.Add("status", resultStatus);
             data.Add("category", NameUtils.GetCategoryName());
             data.Add("fixturename", fixtureName);
+            if (!data.ContainsKey("testname")) data.Add("testname", NameUtils.GetTestMethodName());
             data.Add("testid", NameUtils.GetTestId());
-            data.Add("testname", NameUtils.GetTestMethodName());
             data.Add("testtype", testType);
             data.Add("testduration", testDuration);
+            data.Add("testframeworkrevision", GitInfo.GetStatus());
             Write(data);
         }
 

Tests/DistTestCore/Logs/TestLog.cs

@@ -2,12 +2,11 @@
 {
     public class TestLog : BaseTestLog
     {
-        private readonly string methodName;
         private readonly string fullName;
 
-        public TestLog(string folder, string name = "")
+        public TestLog(string folder, string deployId, string name = "") : base(deployId)
         {
-            methodName = NameUtils.GetTestMethodName(name);
+            var methodName = NameUtils.GetTestMethodName(name);
             fullName = Path.Combine(folder, methodName);
 
             Log($"*** Begin: {methodName}");

Tests/DistTestCore/NameUtils.cs

@@ -5,14 +5,6 @@ namespace DistTestCore
 {
     public static class NameUtils
     {
-        private static readonly string defaultRunId;
-
-        static NameUtils()
-        {
-            var now = DateTime.UtcNow;
-            defaultRunId = now.ToString("yyyyMMdd-hhmmss");
-        }
-
         public static string GetTestMethodName(string name = "")
         {
             if (!string.IsNullOrEmpty(name)) return name;
@@ -48,9 +40,9 @@ namespace DistTestCore
             return GetEnvVar("TESTID", "EnvVar-TESTID-NotSet");
         }
 
-        public static string GetRunId()
+        public static string MakeDeployId()
         {
-            return GetEnvVar("RUNID", defaultRunId);
+            return DateTime.UtcNow.ToString("yyyyMMdd-hhmmss");
         }
 
         private static string GetEnvVar(string name, string defaultValue)

Tests/DistTestCore/TestLifecycle.cs

@@ -11,25 +11,27 @@ namespace DistTestCore
     public class TestLifecycle : IK8sHooks
     {
         private const string TestsType = "dist-tests";
-        private readonly DateTime testStart;
         private readonly EntryPoint entryPoint;
-        private readonly Dictionary<string, string> metadata;
-        private readonly List<RunningContainers> runningContainers = new List<RunningContainers>();
+        private readonly Dictionary<string, string> metadata; 
+        private readonly List<RunningContainers> runningContainers = new();
+        private readonly string deployId;
 
-        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
+        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace, string deployId)
         {
             Log = log;
             Configuration = configuration;
             TimeSet = timeSet;
-            testStart = DateTime.UtcNow;
+            TestStart = DateTime.UtcNow;
 
             entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet);
             metadata = entryPoint.GetPluginMetadata();
             CoreInterface = entryPoint.CreateInterface();
+            this.deployId = deployId;
 
             log.WriteLogTag();
         }
 
+        public DateTime TestStart { get; }
         public TestLog Log { get; }
         public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
@@ -58,10 +60,9 @@ namespace DistTestCore
             return entryPoint.GetPluginMetadata();
         }
 
-        public string GetTestDuration()
+        public TimeSpan GetTestDuration()
         {
-            var testDuration = DateTime.UtcNow - testStart;
-            return Time.FormatDuration(testDuration);
+            return DateTime.UtcNow - TestStart;
         }
 
         public void OnContainersStarted(RunningContainers rc)
@@ -77,11 +78,12 @@ namespace DistTestCore
         public void OnContainerRecipeCreated(ContainerRecipe recipe)
         {
             recipe.PodLabels.Add("tests-type", TestsType);
-            recipe.PodLabels.Add("runid", NameUtils.GetRunId());
+            recipe.PodLabels.Add("deployid", deployId);
             recipe.PodLabels.Add("testid", NameUtils.GetTestId());
             recipe.PodLabels.Add("category", NameUtils.GetCategoryName());
             recipe.PodLabels.Add("fixturename", NameUtils.GetRawFixtureName());
             recipe.PodLabels.Add("testname", NameUtils.GetTestMethodName());
+            recipe.PodLabels.Add("testframeworkrevision", GitInfo.GetStatus());
 
             foreach (var pair in metadata)
             {

Tests/FrameworkTests/FrameworkTests.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="nunit" Version="3.13.3" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\Framework\Utils\Utils.csproj" />
+  </ItemGroup>
+
+</Project>

Tests/FrameworkTests/Utils/TimeTests.cs

@@ -0,0 +1,25 @@
+using NUnit.Framework;
+using Utils;
+
+namespace FrameworkTests.Utils
+{
+    [TestFixture]
+    public class TimeTests
+    {
+        [Test]
+        public void Timespan()
+        {
+            Assert.That(Time.ParseTimespan("10"), Is.EqualTo(TimeSpan.FromSeconds(10)));
+            Assert.That(Time.ParseTimespan("10s"), Is.EqualTo(TimeSpan.FromSeconds(10)));
+            Assert.That(Time.ParseTimespan("10m"), Is.EqualTo(TimeSpan.FromMinutes(10)));
+            Assert.That(Time.ParseTimespan("10d"), Is.EqualTo(TimeSpan.FromDays(10)));
+            Assert.That(Time.ParseTimespan("120s"), Is.EqualTo(TimeSpan.FromSeconds(120)));
+            Assert.That(Time.ParseTimespan("2d14h6m28s"), Is.EqualTo(
+                TimeSpan.FromDays(2) +
+                TimeSpan.FromHours(14) +
+                TimeSpan.FromMinutes(6) +
+                TimeSpan.FromSeconds(28)
+            ));
+        }
+    }
+}

Tools/BiblioTech/AdminChecker.cs

@@ -1,4 +1,5 @@
-using Discord.WebSocket;
+using Discord;
+using Discord.WebSocket;
 
 namespace BiblioTech
 {
@@ -7,6 +8,7 @@ namespace BiblioTech
         private SocketGuild guild = null!;
         private ulong[] adminIds = Array.Empty<ulong>();
         private DateTime lastUpdate = DateTime.MinValue;
+        private ISocketMessageChannel adminChannel = null!;
 
         public void SetGuild(SocketGuild guild)
         {
@@ -20,11 +22,21 @@ namespace BiblioTech
             return adminIds.Contains(userId);
         }
 
-        public bool IsAdminChannel(ISocketMessageChannel channel)
+        public bool IsAdminChannel(IChannel channel)
         {
             return channel.Name == Program.Config.AdminChannelName;
         }
 
+        public ISocketMessageChannel GetAdminChannel()
+        {
+            return adminChannel;
+        }
+
+        public void SetAdminChannel(ISocketMessageChannel adminChannel)
+        {
+            this.adminChannel = adminChannel;
+        }
+
         private bool ShouldUpdate()
         {
             return !adminIds.Any() || (DateTime.UtcNow - lastUpdate) > TimeSpan.FromMinutes(10);

Tools/BiblioTech/BaseCodexCommand.cs

@@ -1,27 +0,0 @@
-using BiblioTech.Options;
-using CodexPlugin;
-using Core;
-
-namespace BiblioTech
-{
-    public abstract class BaseCodexCommand : BaseDeploymentCommand
-    {
-        private readonly CoreInterface ci;
-
-        public BaseCodexCommand(CoreInterface ci)
-        {
-            this.ci = ci;
-        }
-
-        protected override async Task ExecuteDeploymentCommand(CommandContext context, CodexDeployment codexDeployment)
-        {
-            var codexContainers = codexDeployment.CodexInstances.Select(c => c.Containers).ToArray();
-
-            var group = ci.WrapCodexContainers(codexContainers);
-
-            await Execute(context, group);
-        }
-
-        protected abstract Task Execute(CommandContext context, ICodexNodeGroup codexGroup);
-    }
-}

Tools/BiblioTech/BaseCommand.cs

@@ -9,13 +9,7 @@ namespace BiblioTech
         public abstract string Name { get; }
         public abstract string StartingMessage { get; }
         public abstract string Description { get; }
-        public virtual CommandOption[] Options
-        {
-            get
-            {
-                return Array.Empty<CommandOption>();
-            }
-        }
+        public virtual CommandOption[] Options => Array.Empty<CommandOption>();
 
         public async Task SlashCommandHandler(SocketSlashCommand command)
         {
@@ -23,14 +17,24 @@ namespace BiblioTech
 
             try
             {
+                Program.Log.Log($"Responding to '{Name}'");
                 var context = new CommandContext(command, command.Data.Options);
                 await command.RespondAsync(StartingMessage, ephemeral: IsEphemeral(context));
                 await Invoke(context);
             }
             catch (Exception ex)
             {
-                await command.FollowupAsync("Something failed while trying to do that...", ephemeral: true);
-                Console.WriteLine(ex);
+                var msg = "Failed with exception: " + ex;
+                if (IsInAdminChannel(command))
+                {
+                    await command.FollowupAsync(msg.Substring(0, Math.Min(1900, msg.Length)));
+                }
+                else
+                {
+                    await command.FollowupAsync("Something failed while trying to do that...", ephemeral: true);
+                    await Program.AdminChecker.GetAdminChannel().SendMessageAsync(msg);
+                }
+                Program.Log.Error(msg);
             }
         }
 

Tools/BiblioTech/BaseDeploymentCommand.cs

@@ -1,36 +0,0 @@
-using BiblioTech.Options;
-using CodexPlugin;
-
-namespace BiblioTech
-{
-    public abstract class BaseDeploymentCommand : BaseCommand
-    {
-        protected override async Task Invoke(CommandContext context)
-        {
-            var proceed = await OnInvoke(context);
-            if (!proceed) return;
-
-            var deployments = Program.DeploymentFilesMonitor.GetDeployments();
-            if (deployments.Length == 0)
-            {
-                await context.Followup("No deployments are currently available.");
-                return;
-            }
-            if (deployments.Length > 1)
-            {
-                await context.Followup("Multiple deployments are online. I don't know which one to pick!");
-                return;
-            }
-
-            var codexDeployment = deployments.Single();
-            await ExecuteDeploymentCommand(context, codexDeployment);
-        }
-
-        protected abstract Task ExecuteDeploymentCommand(CommandContext context, CodexDeployment codexDeployment);
-
-        protected virtual Task<bool> OnInvoke(CommandContext context)
-        {
-            return Task.FromResult(true);
-        }
-    }
-}

Tools/BiblioTech/BaseGethCommand.cs

@@ -1,27 +1,24 @@
 using BiblioTech.Options;
 using CodexContractsPlugin;
-using CodexPlugin;
-using Core;
 using GethPlugin;
 
 namespace BiblioTech
 {
-    public abstract class BaseGethCommand : BaseDeploymentCommand
+    public abstract class BaseGethCommand : BaseCommand
     {
-        private readonly CoreInterface ci;
-
-        public BaseGethCommand(CoreInterface ci)
+        protected override async Task Invoke(CommandContext context)
         {
-            this.ci = ci;
-        }
+            var gethConnector = GethConnector.GethConnector.Initialize(Program.Log);
 
-        protected override async Task ExecuteDeploymentCommand(CommandContext context, CodexDeployment codexDeployment)
-        {
-            var gethDeployment = codexDeployment.GethDeployment;
-            var contractsDeployment = codexDeployment.CodexContractsDeployment;
+            if (gethConnector == null) return;
+            var gethNode = gethConnector.GethNode;
+            var contracts = gethConnector.CodexContracts;
 
-            var gethNode = ci.WrapGethDeployment(gethDeployment);
-            var contracts = ci.WrapCodexContractsDeployment(gethNode, contractsDeployment);
+            if (!contracts.IsDeployed())
+            {
+                await context.Followup("I'm sorry, the Codex SmartContracts are not currently deployed.");
+                return;
+            }
 
             await Execute(context, gethNode, contracts);
         }

Tools/BiblioTech/BiblioTech.csproj

@@ -10,6 +10,8 @@
   <ItemGroup>
     <PackageReference Include="Discord.Net" Version="3.12.0" />
     <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
+    <ProjectReference Include="..\..\Framework\DiscordRewards\DiscordRewards.csproj" />
+    <ProjectReference Include="..\..\Framework\GethConnector\GethConnector.csproj" />
     <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
   </ItemGroup>
 

Tools/BiblioTech/CommandHandler.cs

@@ -2,6 +2,7 @@
 using Discord.WebSocket;
 using Discord;
 using Newtonsoft.Json;
+using BiblioTech.Rewards;
 
 namespace BiblioTech
 {
@@ -23,18 +24,29 @@ namespace BiblioTech
         {
             var guild = client.Guilds.Single(g => g.Name == Program.Config.ServerName);
             Program.AdminChecker.SetGuild(guild);
+            Program.Log.Log($"Initializing for guild: '{guild.Name}'");
+
+            var roleController = new RoleController(client);
+            var rewardsApi = new RewardsApi(roleController);
+
+            var adminChannels = guild.TextChannels.Where(Program.AdminChecker.IsAdminChannel).ToArray();
+            if (adminChannels == null || !adminChannels.Any()) throw new Exception("No admin message channel");
+            Program.AdminChecker.SetAdminChannel(adminChannels.First());
 
             var builders = commands.Select(c =>
             {
+                var msg = $"Building command '{c.Name}' with options: ";
                 var builder = new SlashCommandBuilder()
                     .WithName(c.Name)
                     .WithDescription(c.Description);
 
                 foreach (var option in c.Options)
                 {
+                    msg += option.Name + " ";
                     builder.AddOption(option.Build());
                 }
 
+                Program.Log.Log(msg);
                 return builder;
             });
 
@@ -48,8 +60,10 @@ namespace BiblioTech
             catch (HttpException exception)
             {
                 var json = JsonConvert.SerializeObject(exception.Errors, Formatting.Indented);
-                Console.WriteLine(json);
+                Program.Log.Error(json);
             }
+
+            rewardsApi.Start();
         }
 
         private async Task SlashCommandHandler(SocketSlashCommand command)

Tools/BiblioTech/Commands/AdminCommand.cs

@@ -1,7 +1,4 @@
 using BiblioTech.Options;
-using CodexPlugin;
-using Core;
-using Newtonsoft.Json;
 
 namespace BiblioTech.Commands
 {
@@ -9,17 +6,16 @@ namespace BiblioTech.Commands
     {
         private readonly ClearUserAssociationCommand clearCommand = new ClearUserAssociationCommand();
         private readonly ReportCommand reportCommand = new ReportCommand();
-        private readonly DeployListCommand deployListCommand = new DeployListCommand();
-        private readonly DeployUploadCommand deployUploadCommand = new DeployUploadCommand();
-        private readonly DeployRemoveCommand deployRemoveCommand = new DeployRemoveCommand();
         private readonly WhoIsCommand whoIsCommand = new WhoIsCommand();
-        private readonly NetInfoCommand netInfoCommand;
-        private readonly DebugPeerCommand debugPeerCommand;
+        private readonly AddSprCommand addSprCommand;
+        private readonly ClearSprsCommand clearSprsCommand;
+        private readonly GetSprCommand getSprCommand;
 
-        public AdminCommand(CoreInterface ci)
+        public AdminCommand(SprCommand sprCommand)
         {
-            netInfoCommand = new NetInfoCommand(ci);
-            debugPeerCommand = new DebugPeerCommand(ci);
+            addSprCommand = new AddSprCommand(sprCommand);
+            clearSprsCommand = new ClearSprsCommand(sprCommand);
+            getSprCommand = new GetSprCommand(sprCommand);
         }
 
         public override string Name => "admin";
@@ -30,12 +26,10 @@ namespace BiblioTech.Commands
         {
             clearCommand,
             reportCommand,
-            deployListCommand,
-            deployUploadCommand,
-            deployRemoveCommand,
             whoIsCommand,
-            netInfoCommand,
-            debugPeerCommand
+            addSprCommand,
+            clearSprsCommand,
+            getSprCommand
         };
 
         protected override async Task Invoke(CommandContext context)
@@ -54,12 +48,10 @@ namespace BiblioTech.Commands
 
             await clearCommand.CommandHandler(context);
             await reportCommand.CommandHandler(context);
-            await deployListCommand.CommandHandler(context);
-            await deployUploadCommand.CommandHandler(context);
-            await deployRemoveCommand.CommandHandler(context);
             await whoIsCommand.CommandHandler(context);
-            await netInfoCommand.CommandHandler(context);
-            await debugPeerCommand.CommandHandler(context);
+            await addSprCommand.CommandHandler(context);
+            await clearSprsCommand.CommandHandler(context);
+            await getSprCommand.CommandHandler(context);
         }
 
         public class ClearUserAssociationCommand : SubCommandOption
@@ -109,108 +101,8 @@ namespace BiblioTech.Commands
                     return;
                 }
 
-                var report = string.Join(Environment.NewLine, Program.UserRepo.GetInteractionReport(user));
-                if (report.Length > 1900)
-                {
-                    var filename = $"user-{user.Username}.log";
-                    await context.FollowupWithAttachement(filename, report);
-                }
-                else
-                {
-                    await context.Followup(report);
-                }
-            }
-        }
-
-        public class DeployListCommand : SubCommandOption
-        {
-            public DeployListCommand()
-                : base("list", "Lists current deployments.")
-            {
-            }
-
-            protected override async Task onSubCommand(CommandContext context)
-            {
-                var deployments = Program.DeploymentFilesMonitor.GetDeployments();
-
-                //todo shows old deployments
-
-                if (!deployments.Any())
-                {
-                    await context.Followup("No deployments available.");
-                    return;
-                }
-
-                var nl = Environment.NewLine;
-                await context.Followup($"Deployments:{nl}{string.Join(nl, deployments.Select(FormatDeployment))}");
-            }
-
-            private string FormatDeployment(CodexDeployment deployment)
-            {
-                var m = deployment.Metadata;
-                return $"'{m.Name}' ({m.StartUtc.ToString("o")})";
-            }
-        }
-
-        public class DeployUploadCommand : SubCommandOption
-        {
-            private readonly FileAttachementOption fileOption = new FileAttachementOption(
-                name: "json",
-                description: "Codex-deployment json to add.",
-                isRequired: true);
-
-            public DeployUploadCommand()
-                : base("add", "Upload a new deployment JSON file.")
-            {
-            }
-
-            public override CommandOption[] Options => new[] { fileOption };
-
-            protected override async Task onSubCommand(CommandContext context)
-            {
-                var file = await fileOption.Parse(context);
-                if (file == null) return;
-
-                var result = await Program.DeploymentFilesMonitor.DownloadDeployment(file);
-                if (result)
-                {
-                    await context.Followup("Success!");
-                }
-                else
-                {
-                    await context.Followup("That didn't work.");
-                }
-            }
-        }
-
-        public class DeployRemoveCommand : SubCommandOption
-        {
-            private readonly StringOption stringOption = new StringOption(
-                name: "name",
-                description: "Name of deployment to remove.",
-                isRequired: true);
-
-            public DeployRemoveCommand()
-                : base("remove", "Removes a deployment file.")
-            {
-            }
-
-            public override CommandOption[] Options => new[] { stringOption };
-
-            protected override async Task onSubCommand(CommandContext context)
-            {
-                var str = await stringOption.Parse(context);
-                if (string.IsNullOrEmpty(str)) return;
-
-                var result = Program.DeploymentFilesMonitor.DeleteDeployment(str);
-                if (result)
-                {
-                    await context.Followup("Success!");
-                }
-                else
-                {
-                    await context.Followup("That didn't work.");
-                }
+                var report = Program.UserRepo.GetInteractionReport(user);
+                await context.Followup(report);
             }
         }
 
@@ -247,118 +139,76 @@ namespace BiblioTech.Commands
             }
         }
 
-        public abstract class AdminDeploymentCommand : SubCommandOption
+        public class AddSprCommand : SubCommandOption
         {
-            private readonly CoreInterface ci;
+            private readonly SprCommand sprCommand;
+            private readonly StringOption stringOption = new StringOption("spr", "Codex SPR", true);
 
-            public AdminDeploymentCommand(CoreInterface ci, string name, string description)
-                : base(name, description)
+            public AddSprCommand(SprCommand sprCommand)
+                : base(name: "addspr",
+               description: "Adds a Codex SPR, to be given to users with '/boot'.")
             {
-                this.ci = ci;
+                this.sprCommand = sprCommand;
             }
 
-            protected async Task OnDeployment(CommandContext context, Func<ICodexNodeGroup, string, Task> action)
-            {
-                var deployment = Program.DeploymentFilesMonitor.GetDeployments().SingleOrDefault();
-                if (deployment == null)
-                {
-                    await context.Followup("No deployment found.");
-                    return;
-                }
+            public override CommandOption[] Options => new[] { stringOption };
 
-                try
+            protected override async Task onSubCommand(CommandContext context)
+            {
+                var spr = await stringOption.Parse(context);
+
+                if (!string.IsNullOrEmpty(spr) )
                 {
-                    var group = ci.WrapCodexContainers(deployment.CodexInstances.Select(i => i.Containers).ToArray());
-                    await action(group, deployment.Metadata.Name);
+                    sprCommand.Add(spr);
+                    await context.Followup("A-OK!");
                 }
-                catch (Exception ex)
+                else
                 {
-                    await context.Followup("Failed to wrap nodes with exception: " + ex);
+                    await context.Followup("SPR is null or empty.");
                 }
             }
         }
 
-        public class NetInfoCommand : AdminDeploymentCommand
+        public class ClearSprsCommand : SubCommandOption
         {
-            public NetInfoCommand(CoreInterface ci)
-                : base(ci, name: "netinfo",
-                      description: "Fetches info endpoints of codex nodes.")
+            private readonly SprCommand sprCommand;
+            private readonly StringOption stringOption = new StringOption("areyousure", "set to 'true' if you are.", true);
+
+            public ClearSprsCommand(SprCommand sprCommand)
+                : base(name: "clearsprs",
+               description: "Clears all Codex SPRs in the bot. Users won't be able to use '/boot' till new ones are added.")
             {
+                this.sprCommand = sprCommand;
+            }
+
+            public override CommandOption[] Options => new[] { stringOption };
+
+            protected override async Task onSubCommand(CommandContext context)
+            {
+                var areyousure = await stringOption.Parse(context);
+
+                if (areyousure != "true") return;
+
+                sprCommand.Clear();
+                await context.Followup("Cleared all SPRs.");
+            }
+        }
+
+        public class GetSprCommand: SubCommandOption
+        {
+            private readonly SprCommand sprCommand;
+
+            public GetSprCommand(SprCommand sprCommand)
+                : base(name: "getsprs",
+               description: "Shows all Codex SPRs in the bot.")
+            {
+                this.sprCommand = sprCommand;
             }
 
             protected override async Task onSubCommand(CommandContext context)
             {
-                await OnDeployment(context, async (group, name) =>
-                {
-                    var nl = Environment.NewLine;
-                    var content = new List<string>
-                    {
-                        $"{DateTime.UtcNow.ToString("o")} - {group.Count()} Codex nodes."
-                    };
-
-                    foreach (var node in group)
-                    {
-                        try
-                        {
-                            var info = node.GetDebugInfo();
-                            var json = JsonConvert.SerializeObject(info, Formatting.Indented);
-                            var jsonInsert = $"{nl}```{nl}{json}{nl}```{nl}";
-                            content.Add($"Node '{node.GetName()}' responded with {jsonInsert}");
-                        }
-                        catch (Exception ex)
-                        {
-                            content.Add($"Node '{node.GetName()}' failed to respond with exception: " + ex);
-                        }
-                    }
-
-                    var filename = $"netinfo-{NoWhitespaces(name)}.log";
-                    await context.FollowupWithAttachement(filename, string.Join(nl, content.ToArray()));
-                });
+                await context.Followup("SPRs: " + string.Join(", ", sprCommand.Get().Select(s => $"'{s}'")));
             }
         }
-
-        public class DebugPeerCommand : AdminDeploymentCommand
-        {
-            private readonly StringOption peerIdOption = new StringOption("peerid", "id of peer to try and reach.", true);
-
-            public DebugPeerCommand(CoreInterface ci)
-                : base(ci, name: "debugpeer",
-                      description: "Calls debug/peer on each codex node.")
-            {
-            }
-
-            public override CommandOption[] Options => new[] { peerIdOption };
-
-            protected override async Task onSubCommand(CommandContext context)
-            {
-                var peerId = await peerIdOption.Parse(context);
-                if (string.IsNullOrEmpty(peerId)) return;
-
-                await OnDeployment(context, async (group, name) =>
-                {
-                    await context.Followup($"Calling debug/peer for '{peerId}' on {group.Count()} Codex nodes.");
-                    foreach (var node in group)
-                    {
-                        try
-                        {
-                            var info = node.GetDebugPeer(peerId);
-                            var nl = Environment.NewLine;
-                            var json = JsonConvert.SerializeObject(info, Formatting.Indented);
-                            var jsonInsert = $"{nl}```{nl}{json}{nl}```{nl}";
-                            await context.Followup($"Node '{node.GetName()}' responded with {jsonInsert}");
-                        }
-                        catch (Exception ex)
-                        {
-                            await context.Followup($"Node '{node.GetName()}' failed to respond with exception: " + ex);
-                        }
-                    }
-                });
-            }
-        }
-
-        private static string NoWhitespaces(string s)
-        {
-            return s.Replace(" ", "-");
-        }
     }
 }