Sets up continuous test for fetch connectivity.

.github/workflows/dist-tests.yaml

@@ -1,40 +1,45 @@
-name: Run Dist Tests
+name: Dist Tests
 
 
 on:
+  # push:
+  #   branches:
+  #     - master
+  #   tags:
+  #     - 'v*.*.*'
+  #   paths-ignore:
+  #     - '**/*.md'
+  #     - '.gitignore'
+  #     - 'docker/**'
+  #     - '!docker/job.yaml'
   workflow_dispatch:
     inputs:
+      branch:
+        description: Branch (master)
+        required: false
+        type: string
       source:
         description: Repository with tests (current)
         required: false
         type: string
-      branch:
-        description: Branch with tests (master)
-        required: false
-        type: string
       nameprefix:
-        description: Runner name prefix (d-tests-runner)
+        description: Runner prefix (cs-codex-dist-tests)
         required: false
         type: string
       namespace:
-        description: Runner namespace (default)
-        required: false
-        type: string
-      command:
-        description: Command (dotnet test Tests/CodexTests)
+        description: Runner namespace (cs-codex-dist-tests)
         required: false
         type: string
 
 
 env:
-  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
   BRANCH: ${{ github.ref_name }}
-  NAMEPREFIX: d-tests-runner
-  NAMESPACE: default
-  COMMAND: dotnet test Tests/CodexTests
-  JOB_MANIFEST: docker/dist-tests-job.yaml
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  NAMEPREFIX: cs-codex-dist-tests
+  NAMESPACE: cs-codex-dist-tests
+  JOB_MANIFEST: docker/job.yaml
   KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
-  KUBE_VERSION: v1.28.2
+  KUBE_VERSION: v1.26.1
 
 
 jobs:
@@ -43,21 +48,19 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
       - name: Variables
         run: |
-          echo "COMMAND=$(jq -c 'split(" ")' <<< '"'$COMMAND'"')" >>"$GITHUB_ENV"
-          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
-          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
           [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
           [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
           [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
           [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ inputs.command }}" ]] && COMMAND="${{ inputs.command }}" || COMMAND="${{ env.COMMAND }}"
+          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
 
       - name: Kubectl - Install ${{ env.KUBE_VERSION }}
-        uses: azure/setup-kubectl@v4
+        uses: azure/setup-kubectl@v3
         with:
           version: ${{ env.KUBE_VERSION }}
 
@@ -69,15 +72,3 @@ jobs:
       - name: Kubectl - Create Job
         run: |
           envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
-
-      - name: Tests Identification
-        run: |
-          echo "----"
-          echo "Repository: ${{ env.SOURCE }}"
-          echo "Branch: ${{ env.BRANCH }}"
-          echo "Runner job: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
-          echo "Runner namespace: ${{ env.NAMESPACE }}"
-          echo "----"
-          echo "Tests runid: ${{ env.RUNID }}"
-          echo "Tests command: `jq -r '. | join(" ")' <<< '${{ env.COMMAND }}'`"
-          echo "----"

.github/workflows/docker-discordbot.yml

@@ -1,28 +0,0 @@
-name: Docker - Discord Bot
-
-
-on:
-  push:
-    branches:
-      - master
-    tags:
-      - 'v*.*.*'
-    paths:
-      - 'Tools/BiblioTech/**'
-      - '!Tools/BiblioTech/docker/docker-compose.yaml'
-      - 'Framework/**'
-      - 'ProjectPlugins/**'
-      - .github/workflows/docker-discordbot.yml
-      - .github/workflows/docker-reusable.yml
-  workflow_dispatch:
-
-
-jobs:
-  build-and-push:
-    name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
-    with:
-      docker_file: Tools/BiblioTech/docker/Dockerfile
-      docker_repo: codexstorage/codex-discordbot
-    secrets: inherit
-

.github/workflows/docker-reusable.yml

@@ -1,4 +1,4 @@
-name: Reusable - Docker
+name: Docker - Reusable
 
 
 on:
@@ -66,26 +66,26 @@ jobs:
       PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
 
       - name: Docker - Meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v4
         with:
           images: ${{ env.DOCKER_REPO }}
 
       - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
 
       - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Docker - Build and Push by digest
         id: build
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v4
         with:
           context: .
           file: ${{ env.DOCKER_FILE }}
@@ -101,9 +101,9 @@ jobs:
           touch "/tmp/digests/${digest#sha256:}"
 
       - name: Docker - Upload digest
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3
         with:
-          name: digests-${{ matrix.target.arch }}
+          name: digests
           path: /tmp/digests/*
           if-no-files-found: error
           retention-days: 1
@@ -139,18 +139,17 @@ jobs:
           fi
 
       - name: Docker - Download digests
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3
         with:
-          pattern: digests-*
-          merge-multiple: true
+          name: digests
           path: /tmp/digests
 
       - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
 
       - name: Docker - Meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v4
         with:
           images: ${{ env.DOCKER_REPO }}
           flavor: |
@@ -162,7 +161,7 @@ jobs:
             type=sha,enable=${{ env.TAG_SHA }}
 
       - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/docker-rewarder.yml

@@ -1,26 +0,0 @@
-name: Docker - Rewarder Bot
-
-on:
-  push:
-    branches:
-      - master
-    tags:
-      - 'v*.*.*'
-    paths:
-      - 'Tools/TestNetRewarder/**'
-      - '!Tools/TestNetRewarder/docker/docker-compose.yaml'
-      - 'Framework/**'
-      - 'ProjectPlugins/**'
-      - .github/workflows/docker-rewarder.yml
-      - .github/workflows/docker-reusable.yml
-  workflow_dispatch:
-
-jobs:
-  build-and-push:
-    name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
-    with:
-      docker_file: Tools/TestNetRewarder/docker/Dockerfile
-      docker_repo: codexstorage/codex-rewarderbot
-    secrets: inherit
-

.github/workflows/docker.yml

@@ -1,4 +1,4 @@
-name: Docker - Runner
+name: Docker
 
 
 on:
@@ -10,7 +10,7 @@ on:
     paths:
       - docker/Dockerfile
       - docker/docker-entrypoint.sh
-      - .github/workflows/docker-runner.yml
+      - .github/workflows/docker.yml
       - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 

.github/workflows/report-HoldMyBeerTest.yaml

@@ -1,20 +0,0 @@
-name: Report - HoldMyBeerTest
-
-
-on:
-  # schedule:
-  #   - cron: '30 1 */2 * *'
-  workflow_dispatch:
-
-jobs:
-  run_tests:
-    name: Call runner
-    uses: ./.github/workflows/run-continuous-tests.yaml
-    with:
-      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
-      branch: master
-      nameprefix: c-tests-report-holdmybeertest-48h
-      tests_target_duration: 48h
-      tests_filter: HoldMyBeerTest
-      tests_cleanup: true
-    secrets: inherit

.github/workflows/report-PeersTest.yaml

@@ -1,20 +0,0 @@
-name: Report - PeersTest
-
-
-on:
-  # schedule:
-  #   - cron: '30 1 */2 * *'
-  workflow_dispatch:
-
-jobs:
-  run_tests:
-    name: Call runner
-    uses: ./.github/workflows/run-continuous-tests.yaml
-    with:
-      source: ${{ format('{0}/{1}', github.server_url, github.repository) }}
-      branch: master
-      nameprefix: c-tests-report-peerstest-48h
-      tests_target_duration: 48h
-      tests_filter: PeersTest
-      tests_cleanup: true
-    secrets: inherit

.github/workflows/run-continuous-tests.yaml

@@ -1,170 +0,0 @@
-name: Run Continuous Tests
-
-
-on:
-  workflow_dispatch:
-    inputs:
-      source:
-        description: Repository with tests (current)
-        required: false
-        type: string
-      branch:
-        description: Branch with tests (master)
-        required: false
-        type: string
-      codexdockerimage:
-        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
-        required: false
-        type: string
-      nameprefix:
-        description: Resources prefix (c-tests)
-        required: false
-        type: string
-      tests_target_duration:
-        description: Runner target duration (2d)
-        required: false
-        type: string
-      tests_filter:
-        description: Runner tests filter ("")
-        required: false
-        type: string
-      tests_cleanup:
-        description: Runner tests cleanup
-        required: false
-        type: boolean
-        default: true
-  workflow_call:
-    inputs:
-      source:
-        description: Repository with tests (current)
-        required: false
-        type: string
-      branch:
-        description: Branch with tests (master)
-        required: false
-        type: string
-      codexdockerimage:
-        description: Codex Docker image (codexstorage/nim-codex:latest-dist-tests)
-        required: false
-        type: string
-      nameprefix:
-        description: Resources prefix (c-tests)
-        required: false
-        type: string
-      tests_target_duration:
-        description: Runner target duration (2d)
-        required: false
-        type: string
-      tests_filter:
-        description: Runner tests filter ("")
-        required: false
-        type: string
-      tests_cleanup:
-        description: Runner tests cleanup
-        required: false
-        type: boolean
-        default: true
-      workflow_source:
-        description: Workflow source
-        required: false
-        type: string
-        default: ''
-
-
-env:
-  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
-  BRANCH: ${{ github.ref_name }}
-  CODEXDOCKERIMAGE: codexstorage/nim-codex:latest-dist-tests
-  NAMEPREFIX: c-tests
-  NAMESPACE: default
-  TESTS_TARGET_DURATION: 2d
-  TESTS_FILTER: ""
-  TESTS_CLEANUP: true
-  JOB_MANIFEST: docker/continuous-tests-job.yaml
-  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
-  KUBE_VERSION: v1.28.2
-
-
-jobs:
-  run_tests:
-    name: Run Continuous Tests ${{ inputs.tests_filter }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ inputs.workflow_source }}
-
-      - name: Variables
-        run: |
-          RUNID=$(date +%Y%m%d-%H%M%S)
-          echo "RUNID=${RUNID}" >> $GITHUB_ENV
-          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
-          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
-          [[ -n "${{ inputs.codexdockerimage }}" ]] && echo "CODEXDOCKERIMAGE=${{ inputs.codexdockerimage }}" >>"$GITHUB_ENV" || echo "CODEXDOCKERIMAGE=${{ env.CODEXDOCKERIMAGE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ inputs.nameprefix }}" ]] && NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ inputs.nameprefix }}`" || NAMEPREFIX="`awk '{ print tolower($0) }' <<< ${{ env.NAMEPREFIX }}`"
-          echo "NAMEPREFIX=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
-          echo "DEPLOYMENT_NAMESPACE=${NAMEPREFIX}-${RUNID}" >>"$GITHUB_ENV"
-          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
-          [[ -n "${{ inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
-          [[ -n "${{ inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
-          [[ -n "${{ inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
-
-      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
-        uses: azure/setup-kubectl@v4
-        with:
-          version: ${{ env.KUBE_VERSION }}
-
-      - name: Kubectl - Kubeconfig
-        run: |
-          mkdir -p "${HOME}"/.kube
-          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
-
-      - name: Kubectl - Create Job
-        run: |
-          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
-
-      - name: Tests Identification
-        run: |
-          echo "----"
-          echo "Repository: ${{ env.SOURCE }}"
-          echo "Branch: ${{ env.BRANCH }}"
-          echo "Runner job: ${{ env.NAMEPREFIX }}"
-          echo "Runner pod: `kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -ojsonpath='{.items[0].metadata.name}'`"
-          echo "Runner namespace: ${{ env.NAMESPACE }}"
-          echo "----"
-          echo "Tests runid: ${{ env.RUNID }}"
-          echo "Tests codexdockerimage: ${{ env.CODEXDOCKERIMAGE }}"
-          echo "Tests namespace: ${{ env.DEPLOYMENT_NAMESPACE }}"
-          echo "Tests duration: ${{ env.TESTS_TARGET_DURATION }}"
-          echo "Tests filter: ${{ env.TESTS_FILTER }}"
-          echo "Tests cleanup: ${{ env.TESTS_CLEANUP }}"
-          echo "----"
-
-      - name: Show Runner logs
-        if: false
-        run: |
-          # Variables
-          # We need more than 300 seconds because Auto Scaler may take 3 minutes to tun a node 
-          duration=600
-          namespace="${{ env.NAMESPACE }}"
-          pod=$(kubectl get pod --selector job-name=${{ env.NAMEPREFIX }} -o jsonpath="{.items[0].metadata.name}")
-
-          # Check Pod status
-          WAIT=120
-          SECONDS=0
-          sleep=1
-          while (( SECONDS < WAIT )); do
-            phase=$(kubectl get pod ${pod} -n ${namespace} -o jsonpath="{.status.phase}")
-            [[ "${phase}" == "Running" ]] && { echo "Pod $pod is in $phase state - Get the logs"; break; } || { echo "Pod $pod is in $phase state - Retry in $sleep seconds / $((WAIT - SECONDS))"; }
-            sleep $sleep
-          done
-
-          # Get logs
-          timeout $duration \
-            kubectl logs $pod \
-              -n $namespace \
-              -f \
-              --tail=-1 \
-              --timestamps || true

ArgsUniform/ArgsUniform.cs

@@ -233,8 +233,7 @@ namespace ArgsUniform
 
         private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
         {
-            var s = value.ToString();
-            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
+            if (value != null)
             {
                 uniformProperty.SetValue(result, true);
             }

CONTRIBUTINGPLUGINS.MD

@@ -1,86 +0,0 @@
-# Distributed System Tests for Nim-Codex
-
-## Contributing plugins
-The testing framework was created for testing Codex. However, it's been designed such that other distributed/containerized projects can 'easily' be added. In order to add your project to the framework you must:
-1. Create a library assembly in the project plugins folder.
-1. It must contain a type that implements the `IProjectPlugin` interface from the `Core` assembly.
-1. If your plugin wants to expose any specific methods or objects to the code using the framework (the tests and tools), it must implement extensions for the `CoreInterface` type.
-
-## Constructors & Tools
-Your implementation of `IProjectPlugin` must have a public constructor with a single argument of type `IPluginTools`, for example:
-```C#
-   public class MyPlugin : IProjectPlugin
-   {
-      public MyPlugin(IPluginTools tools)
-      {
-         ...
-      }
-
-      ...
-   }
-```
-
-`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients for containers. (Without having to figure out addresses manually.)
-
-## Plugin Interfaces
-The `IProjectPlugin` interface requires the implementation of two methods.
-1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version information at this time if applicable.
-1. `Decommission` - Should your plugin have any active system resources, free them in this method.
-
-There are a few optional interfaces your plugin may choose to implement. The framework will automatically use these interfaces.
-1. `IHasLogPrefix` - Implementing this interface allows you to provide a string with will be prepended to all log statements made by your plugin.
-1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by code that uses your plugin.
-
-## Core Interface
-Any functionality your plugin wants to expose to code which uses the framework will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type:
-```C#
-   public static class CoreInterfaceExtensions
-   {
-      public static MyPluginReturnType DoSomethingCool(this CoreInterface ci, string someArgument)
-      {
-         return Plugin(ci).SomethingCool(someArgument);
-      }
-
-      private static MyPlugin Plugin(CoreInterface ci)
-      {
-         return ci.GetPlugin<MyPlugin>();
-      }
-   }
-```
-
-While technically you can build whatever you like on top of the `CoreInterface` and your own plugin types, I recommend that you follow the approach explained below.
-
-## Deploying, Wrapping, and Starting
-When building a plugin, it is important to make as few assumptions as possible about how it will be used by whoever is going to use the framework. For this reason, I recommend you expose three kinds of methods using your `CoreInterface` extensions:
-1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning containers as a result. If your project requires additional information, you can create a new class type to contain both it and the containers created.
-1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain.
-1. Start - This kind of method does both, simply calling a Deploy method first, then a Wrap method, and returns the result.
-
-Here's an example:
-```C#
-public static class CoreInterfaceExtensions
-   {
-      public static RunningContainers DeployMyProject(this CoreInterface ci, string someArgument)
-      {
-         // `RunningContainers` is a framework type. It contains all necessary information about a deployed container. It is serializable.
-         // Should you need to return any additional information, create a new type that contains it as well as the container information. Make sure it is serializable.
-         return Plugin(ci).DeployMyProjectContainer(someArgument); // <-- This method should use the `PluginTools.CreateWorkflow()` tool to deploy a container with a configuration that matches someArguments.
-      }
-
-      public static IMyProjectNode WrapMyProjectContainer(this CoreInterface ci, RunningContainers container)
-      {
-         return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` tool to create an HTTP client for the container, then wrap it in an object that
-         // represents the API of your project.
-      }
-
-      public static IMyProjectNode StartMyProject(this CoreInterface ci, string someArgument)
-      {
-         // Start is now nothing more than a convenience method, combining the previous two.
-         var rc = ci.DeployMyProject(someArgument);
-         return WrapMyProjectContainer(ci, rc);
-      }
-   }
-```
-
-The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing.
-

CONTRIBUTINGTESTS.MD

@@ -1,40 +1,24 @@
 # Distributed System Tests for Nim-Codex
 
 ## Contributing tests
-Do you want to write some tests using this distributed test setup? Great! Here's what you do.
+Do you want to write some tests for Codex using this distributed test setup? Great! Here's what you do.
 
 1. Create a branch. Name it something descriptive, but start it with `tests/` please. [Example: `tests/data-redundancy`.]
-1. Checkout your branch.
-1. Create a new assembly in the `/Tests` folder. This can be an NUnit test assembly or simply a console app.
-1. Add Project references to `Core`, as well as any project plugin you'll be using.
-1. Write tests! Use existing tests for inspiration.
-
-## Tips for writing tests for Codex
-### Transient tests
-1. Add new code files to `Tests/CodexTests`
-1. Inherrit from `CodexDistTest` or `AutoBootstrapDistTest`.
-1. When using `CodexDistTest`:
-   1. You must start your own Codex bootstrap node. You can use `AddCodex(...)` for this.
-   1. When you start other Codex nodes with `AddCodex(...)` you can pass the bootstrap node by adding the `.WithBootstrapNode(...)` option.
-1. When using `AutoBootstrapDistTest`:
-   1. The test-infra creates the bootstrap node for you, and automatically passes it to each Codex node you create in your tests. Handy for keeping your tests clean and to-the-point.
-   1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `CodexDistTest` instead.
-1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
-### Continuous tests
-1. Add new code files to `Tests/CodexContinousTests/Tests`
-1. Inherrit from `ContinuousTest`
-1. Define one or more methods and decorate them with the `[TestMoment(...)]` attribute.
-1. The TestMoment takes a number of seconds as argument. Each moment will be executed by the continuous test runner applying the given seconds as delay. (Non-cumulative. So two moments at T:10 will be executed one after another without delay, in this case the order of execution should not be depended upon.)
-1. Continuous tests automatically receive access to the Codex nodes that the tests are being run against.
-1. Additionally, Continuous tests can start their own transient Codex nodes and bootstrap them against the persistent nodes.
-
-### Tips for either type of test
-1. You can generate files of random test data by calling `GenerateTestFile(...)`.
-1. You can enable access to the Codex node metrics by adding the option `.EnableMetrics()`. Enabling metrics will make the test-infra download and save all Codex metrics in case of a test failure. (The metrics are stored as CSV, in the same location as the test log file.)
-1. You can enable access to the blockchain marketplace by adding the option `.EnableMarketplace(...)`.
-1. Enabling metrics and/or enabling the marketplace takes extra resources from the test-infra and increases the time needed during Codex node setup. Please don't enable these features unless your tests need them.
-1. Tip: Codex nodes can be named. Use the option `WithName(...)` and make reading your test logs a little nicer!
-1. Tip: Commit often.
-
-## Don't forget
-1. Once you're happy with your tests, please create a pull-request and ask a Codex core contributor to review your changes.
+1. Checkout your branch, and decide if your tests will be 'short' tests (minutes to hours), or 'long' tests (hours to days), or both! Create a folder for your tests in the matching folders (`/Tests`, `/LongTests`) and don't worry: You can always move your tests later if you like. [Example, short: `/Tests/DataRedundancy/`, long: `/LongTests/DataRedundancy/`]
+1. Create one or more code files in your new folder, and write some tests! Here are some tips to help you get started. You can always take a look at the example tests found in [`/Tests/BasicTests/ExampleTests.cs`](/Tests/BasicTests/ExampleTests.cs)
+   1. Set up a standard NUnit test fixture.
+   1. Inherrit from `DistTest` or `AutoBootstrapDistTest`.
+   1. When using `DistTest`:
+      1. You must start your own Codex bootstrap node. You can use `SetupCodexBootstrapNode(...)` for this.
+      1. When you start other Codex nodes with `SetupCodexNodes(...)` you can pass the bootstrap node by adding the `.WithBootstrapNode(...)` option.
+   1. When using `AutoBootstrapDistTest`:
+      1. The test-infra creates the bootstrap node for you, and automatically passes it to each Codex node you create in your tests. Handy for keeping your tests clean and to-the-point.
+      1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `DistTest` instead.
+   1. You can generate files of random test data by calling `GenerateTestFile(...)`.
+   1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
+   1. You can enable access to the Codex node metrics by adding the option `.EnableMetrics()`. Enabling metrics will make the test-infra download and save all Codex metrics in case of a test failure. (The metrics are stored as CSV, in the same location as the test log file.)
+   1. You can enable access to the blockchain marketplace by adding the option `.EnableMarketplace(...)`.
+   1. Enabling metrics and/or enabling the marketplace takes extra resources from the test-infra and increases the time needed during Codex node setup. Please don't enable these features unless your tests need them.
+   1. Tip: Codex nodes can be named. Use the option `WithName(...)` and make reading your test logs a little nicer!
+   1. Tip: Commit often.
+1. Once you're happy with your tests, please create a pull-request and ask (another) Codex core contributor to review your changes.

CodexNetDeployer/CodexNetDeployer.csproj

@@ -1,14 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
+    <OutputType>Exe</OutputType>
     <TargetFramework>net7.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\Framework\Core\Core.csproj" />
-    <ProjectReference Include="..\CodexPlugin\CodexPlugin.csproj" />
+    <ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
+    <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
   </ItemGroup>
 
 </Project>

CodexNetDeployer/CodexNodeStarter.cs

@@ -0,0 +1,126 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using KubernetesWorkflow;
+
+namespace CodexNetDeployer
+{
+    public class CodexNodeStarter
+    {
+        private readonly Configuration config;
+        private readonly TestLifecycle lifecycle;
+        private readonly GethStartResult gethResult;
+        private string bootstrapSpr = "";
+        private int validatorsLeft;
+
+        public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
+        {
+            this.config = config;
+            this.lifecycle = lifecycle;
+            this.gethResult = gethResult;
+            validatorsLeft = numberOfValidators;
+        }
+
+        public CodexNodeStartResult? Start(int i)
+        {
+            Console.Write($" - {i} = ");
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var workflowStartup = new StartupConfig();
+            workflowStartup.Add(gethResult);
+            workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));
+            workflowStartup.NameOverride = GetCodexContainerName(i);
+
+            var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
+
+            var container = containers.Containers.First();
+            var codexAccess = new CodexAccess(lifecycle.Log, container, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(container));
+            var account = gethResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[i];
+            var tokenAddress = gethResult.MarketplaceNetwork.Marketplace.TokenAddress;
+            var marketAccess = new MarketplaceAccess(lifecycle, gethResult.MarketplaceNetwork, account, codexAccess);
+
+            try
+            {
+                var debugInfo = codexAccess.GetDebugInfo();
+                if (!string.IsNullOrWhiteSpace(debugInfo.spr))
+                {
+                    Console.Write("Online\t");
+
+                    var interaction = gethResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
+                    interaction.MintTestTokens(new[] { account.Account }, config.InitialTestTokens, tokenAddress);
+                    Console.Write("Tokens minted\t");
+
+                    var response = marketAccess.MakeStorageAvailable(
+                        totalSpace: config.StorageSell!.Value.MB(),
+                        minPriceForTotalSpace: config.MinPrice.TestTokens(),
+                        maxCollateral: config.MaxCollateral.TestTokens(),
+                        maxDuration: TimeSpan.FromSeconds(config.MaxDuration));
+
+                    if (!string.IsNullOrEmpty(response))
+                    {
+                        Console.Write("Storage available\tOK" + Environment.NewLine);
+
+                        if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
+                        validatorsLeft--;
+                        return new CodexNodeStartResult(workflow, container, codexAccess);
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine("Exception:" + ex.ToString());
+            }
+
+            Console.Write("Unknown failure. Downloading container log." + Environment.NewLine);
+            lifecycle.DownloadLog(container);
+
+            return null;
+        }
+
+        private string GetCodexContainerName(int i)
+        {
+            if (i == 0) return "BOOTSTRAP";
+            return "CODEX" + i;
+        }
+
+        private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft)
+        {
+            var codexStart = new CodexStartupConfig(config.CodexLogLevel);
+
+            if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr;
+            codexStart.StorageQuota = config.StorageQuota!.Value.MB();
+            var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
+            marketplaceConfig.AccountIndexOverride = i;
+            codexStart.MarketplaceConfig = marketplaceConfig;
+            codexStart.MetricsMode = config.Metrics;
+
+            if (config.BlockTTL != Configuration.SecondsIn1Day)
+            {
+                codexStart.BlockTTL = config.BlockTTL;
+            }
+            if (config.BlockMI != Configuration.TenMinutes)
+            {
+                codexStart.BlockMaintenanceInterval = TimeSpan.FromSeconds(config.BlockMI);
+            }
+            if (config.BlockMN != 1000)
+            {
+                codexStart.BlockMaintenanceNumber = config.BlockMN;
+            }
+
+            return codexStart;
+        }
+    }
+
+    public class CodexNodeStartResult
+    {
+        public CodexNodeStartResult(StartupWorkflow workflow, RunningContainer container, CodexAccess access)
+        {
+            Workflow = workflow;
+            Container = container;
+            Access = access;
+        }
+
+        public StartupWorkflow Workflow { get; }
+        public RunningContainer Container { get; }
+        public CodexAccess Access { get; }
+    }
+}

CodexNetDeployer/Configuration.cs

@@ -0,0 +1,111 @@
+using ArgsUniform;
+using DistTestCore.Codex;
+using DistTestCore.Metrics;
+
+namespace CodexNetDeployer
+{
+    public class Configuration
+    {
+        public const int SecondsIn1Day = 24 * 60 * 60;
+        public const int TenMinutes = 10 * 60;
+
+        [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
+        public string KubeConfigFile { get; set; } = "null";
+
+        [Uniform("kube-namespace", "kn", "KUBENAMESPACE", true, "Kubernetes namespace to be used for deployment.")]
+        public string KubeNamespace { get; set; } = string.Empty;
+
+        [Uniform("nodes", "n", "NODES", true, "Number of Codex nodes to be created.")]
+        public int? NumberOfCodexNodes { get; set; }
+
+        [Uniform("validators", "v", "VALIDATORS", true, "Number of Codex nodes that will be validating.")]
+        public int? NumberOfValidators { get; set; }
+
+        [Uniform("storage-quota", "sq", "STORAGEQUOTA", true, "Storage quota in megabytes used by each Codex node.")]
+        public int? StorageQuota { get; set; }
+
+        [Uniform("storage-sell", "ss", "STORAGESELL", true, "Number of megabytes of storage quota to make available for selling.")]
+        public int? StorageSell { get; set; }
+
+        [Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")]
+        public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug;
+
+        [Uniform("test-tokens", "tt", "TESTTOKENS", true, "Initial amount of test-tokens minted for each Codex node.")]
+        public int InitialTestTokens { get; set; } = int.MaxValue;
+
+        [Uniform("min-price", "mp", "MINPRICE", true, "Minimum price for the storage space for which contracts will be accepted.")]
+        public int MinPrice { get; set; }
+
+        [Uniform("max-collateral", "mc", "MAXCOLLATERAL", true, "Maximum collateral that will be placed for the total storage space.")]
+        public int MaxCollateral { get; set; }
+
+        [Uniform("max-duration", "md", "MAXDURATION", true, "Maximum duration in seconds for contracts which will be accepted.")]
+        public int MaxDuration { get; set; }
+
+        [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
+        public int BlockTTL { get; set; } = SecondsIn1Day;
+
+        [Uniform("block-mi", "bmi", "BLOCKMI", false, "Block maintenance interval in seconds. Default is 10 minutes.")]
+        public int BlockMI { get; set; } = TenMinutes;
+
+        [Uniform("block-mn", "bmn", "BLOCKMN", false, "Number of blocks maintained per interval. Default is 1000 blocks.")]
+        public int BlockMN { get; set; } = 1000;
+
+        [Uniform("metrics", "m", "METRICS", false, "[None*, Record, Dashboard]. Determines if metrics will be recorded and if a dashboard service will be created.")]
+        public MetricsMode Metrics { get; set; } = MetricsMode.None;
+
+        [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
+            "set this option to override the label value.")]
+        public string TestsTypePodLabel { get; set; } = "continuous-tests";
+
+        [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment.")]
+        public bool CheckPeerConnection { get; set; } = false;
+       
+        public List<string> Validate()
+        {
+            var errors = new List<string>();
+
+            ForEachProperty(
+                onString: (n, v) => StringIsSet(n, v, errors),
+                onInt: (n, v) => IntIsOverZero(n, v, errors));
+
+            if (NumberOfValidators > NumberOfCodexNodes)
+            {
+                errors.Add($"{nameof(NumberOfValidators)} ({NumberOfValidators}) may not be greater than {nameof(NumberOfCodexNodes)} ({NumberOfCodexNodes}).");
+            }
+            if (StorageSell.HasValue && StorageQuota.HasValue && StorageSell.Value >= StorageQuota.Value)
+            {
+                errors.Add("StorageSell cannot be greater than or equal to StorageQuota.");
+            }
+
+            return errors;
+        }
+
+        private void ForEachProperty(Action<string, string> onString, Action<string, int?> onInt)
+        {
+            var properties = GetType().GetProperties();
+            foreach (var p in properties)
+            {
+                if (p.PropertyType == typeof(string)) onString(p.Name, (string)p.GetValue(this)!);
+                if (p.PropertyType == typeof(int?)) onInt(p.Name, (int?)p.GetValue(this)!);
+                if (p.PropertyType == typeof(int)) onInt(p.Name, (int)p.GetValue(this)!);
+            }
+        }
+
+        private static void IntIsOverZero(string variable, int? value, List<string> errors)
+        {
+            if (value == null || value.Value < 1)
+            {
+                errors.Add($"{variable} must be set and must be greater than 0.");
+            }
+        }
+
+        private static void StringIsSet(string variable, string value, List<string> errors)
+        {
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                errors.Add($"{variable} must be set.");
+            }
+        }
+    }
+}

CodexNetDeployer/Deployer.cs

@@ -0,0 +1,155 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+using Logging;
+
+namespace CodexNetDeployer
+{
+    public class Deployer
+    {
+        private readonly Configuration config;
+        private readonly DefaultTimeSet timeset;
+        private readonly PeerConnectivityChecker peerConnectivityChecker;
+
+        public Deployer(Configuration config)
+        {
+            this.config = config;
+            timeset = new DefaultTimeSet();
+            peerConnectivityChecker = new PeerConnectivityChecker();
+        }
+
+        public CodexDeployment Deploy()
+        {
+            Log("Initializing...");
+            var lifecycle = CreateTestLifecycle();
+
+            Log("Preparing configuration...");
+            // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
+            var setup = new CodexSetup(999, config.CodexLogLevel);
+            setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
+            setup.MetricsMode = config.Metrics;
+
+            Log("Creating Geth instance and deploying contracts...");
+            var gethStarter = new GethStarter(lifecycle);
+            var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
+
+            Log("Geth started. Codex contracts deployed.");
+            Log("Warning: It can take up to 45 minutes for the Geth node to finish unlocking all if its 1000 preconfigured accounts.");
+
+            // It takes a second for the geth node to unlock a single account. Let's assume 3.
+            // We can't start the codex nodes until their accounts are definitely unlocked. So
+            // We wait:
+            Thread.Sleep(TimeSpan.FromSeconds(3.0 * config.NumberOfCodexNodes!.Value));
+
+            Log("Starting Codex nodes...");
+
+            // Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
+            var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
+            var startResults = new List<CodexNodeStartResult>();
+            for (var i = 0; i < config.NumberOfCodexNodes; i++)
+            {
+                var result = codexStarter.Start(i);
+                if (result != null) startResults.Add(result);
+            }
+
+            var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container));
+
+            CheckPeerConnectivity(startResults);
+            CheckContainerRestarts(startResults);
+
+            return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
+        }
+
+        private TestLifecycle CreateTestLifecycle()
+        {
+            var kubeConfig = GetKubeConfig(config.KubeConfigFile);
+
+            var lifecycleConfig = new DistTestCore.Configuration
+            (
+                kubeConfigFile: kubeConfig,
+                logPath: "null",
+                logDebug: false,
+                dataFilesPath: "notUsed",
+                codexLogLevel: config.CodexLogLevel,
+                k8sNamespacePrefix: config.KubeNamespace
+            );
+
+            var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, timeset, string.Empty);
+            DefaultContainerRecipe.TestsType = config.TestsTypePodLabel;
+            DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
+            return lifecycle;
+        }
+
+        private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable<RunningContainer> codexContainers)
+        {
+            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
+
+            Log("Starting metrics service...");
+            var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
+            var prometheusContainer = lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
+
+            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null);
+
+            Log("Starting dashboard service...");
+            var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer, setup);
+            return (prometheusContainer, grafanaStartInfo);
+        }
+
+        private string? GetKubeConfig(string kubeConfigFile)
+        {
+            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
+            return kubeConfigFile;
+        }
+
+        private void CheckPeerConnectivity(List<CodexNodeStartResult> codexContainers)
+        {
+            if (!config.CheckPeerConnection) return;
+
+            Log("Starting peer-connectivity check for deployed nodes...");
+            peerConnectivityChecker.CheckConnectivity(codexContainers);
+            Log("Check passed.");
+        }
+
+        private void CheckContainerRestarts(List<CodexNodeStartResult> startResults)
+        {
+            var crashes = new List<RunningContainer>();
+            foreach (var startResult in startResults)
+            {
+                var watcher = startResult.Workflow.CreateCrashWatcher(startResult.Container);
+                if (watcher.HasContainerCrashed()) crashes.Add(startResult.Container);
+            }
+
+            if (!crashes.Any())
+            {
+                Log("Container restart check passed.");
+            }
+            else
+            {
+                Log($"Deployment failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}");
+                throw new Exception("Deployment failed: One or more containers crashed.");
+            }
+        }
+
+        private DeploymentMetadata CreateMetadata()
+        {
+            return new DeploymentMetadata(
+                kubeNamespace: config.KubeNamespace,
+                numberOfCodexNodes: config.NumberOfCodexNodes!.Value,
+                numberOfValidators: config.NumberOfValidators!.Value,
+                storageQuotaMB: config.StorageQuota!.Value,
+                codexLogLevel: config.CodexLogLevel,
+                initialTestTokens: config.InitialTestTokens,
+                minPrice: config.MinPrice,
+                maxCollateral: config.MaxCollateral,
+                maxDuration: config.MaxDuration,
+                blockTTL: config.BlockTTL,
+                blockMI: config.BlockMI,
+                blockMN: config.BlockMN);
+        }
+
+        private void Log(string msg)
+        {
+            Console.WriteLine(msg);
+        }
+    }
+}

CodexNetDeployer/PeerConnectivityChecker.cs

@@ -0,0 +1,34 @@
+using DistTestCore.Helpers;
+using Logging;
+
+namespace CodexNetDeployer
+{
+    public class PeerConnectivityChecker
+    {
+        public void CheckConnectivity(List<CodexNodeStartResult> startResults)
+        {
+            var log = new ConsoleLog();
+            var checker = new PeerConnectionTestHelpers(log);
+            var access = startResults.Select(r => r.Access);
+
+            checker.AssertFullyConnected(access);
+        }
+    }
+
+    public class ConsoleLog : BaseLog
+    {
+        public ConsoleLog() : base(false)
+        {
+        }
+
+        protected override string GetFullName()
+        {
+            return "CONSOLE";
+        }
+
+        public override void Log(string message)
+        {
+            Console.WriteLine(message);
+        }
+    }
+}

CodexNetDeployer/Program.cs

@@ -1,5 +1,8 @@
 using ArgsUniform;
 using CodexNetDeployer;
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
 using Newtonsoft.Json;
 using Configuration = CodexNetDeployer.Configuration;
 
@@ -23,29 +26,34 @@ public class Program
             return;
         }
 
-        var deployer = new Deployer(config);
-        deployer.AnnouncePlugins();
+        Console.WriteLine("Using images:" + nl +
+            $"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl +
+            $"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl +
+            $"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl +
+            $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl +
+            $"\tGrafana image: '{new GrafanaContainerRecipe().Image}'" + nl);
 
-        if (config.IsPublicTestNet || !args.Any(a => a == "-y"))
+        if (!args.Any(a => a == "-y"))
         {
-            if (config.IsPublicTestNet) Console.WriteLine("Deployment is configured as public testnet.");
             Console.WriteLine("Does the above config look good? [y/n]");
             if (Console.ReadLine()!.ToLowerInvariant() != "y") return;
-            if (config.IsPublicTestNet) Console.WriteLine("You better be right about that, cause it's going live right now.");
-            else Console.WriteLine("I think so too.");
+            Console.WriteLine("I think so too.");
         }
 
+        var deployer = new Deployer(config);
         var deployment = deployer.Deploy();
 
-        Console.WriteLine($"Writing deployment file '{config.DeployFile}'...");
-        File.WriteAllText(config.DeployFile, JsonConvert.SerializeObject(deployment, Formatting.Indented));
+        Console.WriteLine("Writing codex-deployment.json...");
+
+        File.WriteAllText("codex-deployment.json", JsonConvert.SerializeObject(deployment, Formatting.Indented));
+
         Console.WriteLine("Done!");
     }
 
     private static void PrintHelp()
     {
         var nl = Environment.NewLine;
-        Console.WriteLine("CodexNetDeployer allows you to deploy multiple Codex nodes in a Kubernetes cluster. " +
+        Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
             "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
             "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
     }

CodexNetDeployer/deploy-continuous-testnet.sh

@@ -0,0 +1,16 @@
+dotnet run \
+    --kube-config=/opt/kubeconfig.yaml \
+    --kube-namespace=codex-continuous-tests \
+    --nodes=5 \
+    --validators=3 \
+    --log-level=Trace \
+    --storage-quota=2048 \
+    --storage-sell=1024 \
+    --min-price=1024 \
+    --max-collateral=1024 \
+    --max-duration=3600000 \
+    --block-ttl=180 \
+    --block-mi=120 \
+    --block-mn=10000 \
+    --metrics=Dashboard \
+    --check-connect=1

CodexNetDownloader/CodexNetDownloader.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
+    <ProjectReference Include="..\ContinuousTests\ContinuousTests.csproj" />
+    <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+  </ItemGroup>
+
+</Project>

CodexNetDownloader/Configuration.cs

@@ -0,0 +1,19 @@
+using ArgsUniform;
+using DistTestCore.Codex;
+
+namespace CodexNetDownloader
+{
+    public class Configuration
+    {
+        [Uniform("output-path", "o", "OUTPUT", true, "Path where files will be written.")]
+        public string OutputPath { get; set; } = "output";
+
+        [Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")]
+        public string CodexDeploymentJson { get; set; } = string.Empty;
+
+        [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
+        public string KubeConfigFile { get; set; } = "null";
+
+        public CodexDeployment CodexDeployment { get; set; } = null!;
+    }
+}

CodexNetDownloader/Program.cs

@@ -0,0 +1,48 @@
+using ArgsUniform;
+using ContinuousTests;
+using DistTestCore;
+using DistTestCore.Codex;
+using Logging;
+using Newtonsoft.Json;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        var nl = Environment.NewLine;
+        Console.WriteLine("CodexNetDownloader" + nl);
+
+        var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(PrintHelp, args);
+        var config = uniformArgs.Parse(true);
+
+        config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
+
+        if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
+
+        var k8sFactory = new K8sFactory();
+        var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
+
+        foreach (var container in config.CodexDeployment.CodexContainers)
+        {
+            lifecycle.DownloadLog(container);
+        }
+
+        Console.WriteLine("Done!");
+    }
+
+    private static CodexDeployment ParseCodexDeploymentJson(string filename)
+    {
+        var d = JsonConvert.DeserializeObject<CodexDeployment>(File.ReadAllText(filename))!;
+        if (d == null) throw new Exception("Unable to parse " + filename);
+        return d;
+    }
+
+    private static void PrintHelp()
+    {
+        var nl = Environment.NewLine;
+        Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
+
+        Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
+            "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
+    }
+}

ContinuousTests/CodexAccessFactory.cs

@@ -0,0 +1,20 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class CodexAccessFactory
+    {
+        public CodexAccess[] Create(Configuration config, RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
+        {
+            return containers.Select(container =>
+            {
+                var address = container.ClusterExternalAddress;
+                if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
+                return new CodexAccess(log, container, timeSet, address);
+            }).ToArray();
+        }
+    }
+}

ContinuousTests/Configuration.cs

@@ -1,5 +1,6 @@
 using ArgsUniform;
-using CodexPlugin;
+using DistTestCore;
+using DistTestCore.Codex;
 using Newtonsoft.Json;
 
 namespace ContinuousTests
@@ -15,7 +16,7 @@ namespace ContinuousTests
         [Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")]
         public string CodexDeploymentJson { get; set; } = string.Empty;
 
-        [Uniform("keep", "k", "KEEP", false, "Set to 1 or 'true' to retain logs of successful tests.")]
+        [Uniform("keep", "k", "KEEP", false, "Set to '1' to retain logs of successful tests.")]
         public bool KeepPassedTestLogs { get; set; } = false;
 
         [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
@@ -24,20 +25,12 @@ namespace ContinuousTests
         [Uniform("stop", "s", "STOPONFAIL", false, "If greater than zero, runner will stop after this many test failures and download all cluster container logs. 0 by default.")]
         public int StopOnFailure { get; set; } = 0;
 
-        [Uniform("target-duration", "td", "TARGETDURATION", false, "If set, runner will run for this length of time before stopping. Supports seconds, or '1d2h3m4s' format.")]
-        public string TargetDurationSeconds { get; set; } = string.Empty;
-
-        [Uniform("filter", "f", "FILTER", false, "If set, runs only tests whose names contain any of the filter strings. Comma-separated. Case sensitive.")]
-        public string Filter { get; set; } = string.Empty;
-
-        [Uniform("cleanup", "cl", "CLEANUP", false, "If set to 1 or 'true', the kubernetes namespace will be deleted after the test run has finished.")]
-        public bool Cleanup { get; set; } = false;
-
-        [Uniform("full-container-logs", "fcl", "FULLCONTAINERLOGS", false, "If set to 1 or 'true', container logs downloaded on test failure will download from" +
-            " the timestamp of the start of the network deployment. Otherwise, logs will start from the test start timestamp.")]
-        public bool FullContainerLogs { get; set; } = false;
+        [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
+        public bool DownloadContainerLogs { get; set; } = false;
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
+
+        public RunnerLocation RunnerLocation { get; set; }
     }
 
     public class ConfigLoader
@@ -47,7 +40,10 @@ namespace ContinuousTests
             var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
 
             var result = uniformArgs.Parse(true);
+            
             result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
+            result.RunnerLocation = RunnerLocationUtils.DetermineRunnerLocation(result.CodexDeployment.CodexContainers.First());
+
             return result;
         }
         
@@ -62,7 +58,10 @@ namespace ContinuousTests
         {
             var nl = Environment.NewLine;
             Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
-                "The tests will run in an endless loop unless otherwise specified." + nl);
+                "The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
+
+            Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
+                "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
         }
     }
 }

ContinuousTests/ContinuousTest.cs

@@ -0,0 +1,119 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using DistTestCore.Logs;
+using KubernetesWorkflow;
+using Logging;
+
+namespace ContinuousTests
+{
+    public abstract class ContinuousTestLongTimeouts : ContinuousTest
+    {
+        public override ITimeSet TimeSet => new LongTimeSet();
+    }
+
+    public abstract class ContinuousTest
+    {
+        protected const int Zero = 0;
+        protected const int MinuteOne = 60;
+        protected const int MinuteFive = MinuteOne * 5;
+        protected const int HourOne = MinuteOne * 60;
+        protected const int HourThree = HourOne * 3;
+        protected const int DayOne = HourOne * 24;
+        protected const int DayThree = DayOne * 3;
+
+        private const string UploadFailedMessage = "Unable to store block";
+
+        public void Initialize(CodexAccess[] nodes, BaseLog log, FileManager fileManager, Configuration configuration, CancellationToken cancelToken)
+        {
+            Nodes = nodes;
+            Log = log;
+            FileManager = fileManager;
+            Configuration = configuration;
+            CancelToken = cancelToken;
+
+            if (nodes != null)
+            {
+                NodeRunner = new NodeRunner(Nodes, configuration, TimeSet, Log, CustomK8sNamespace, EthereumAccountIndex);
+            }
+            else
+            {
+                NodeRunner = null!;
+            }
+        }
+
+        public CodexAccess[] Nodes { get; private set; } = null!;
+        public BaseLog Log { get; private set; } = null!;
+        public IFileManager FileManager { get; private set; } = null!;
+        public Configuration Configuration { get; private set; } = null!;
+        public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
+        public CancellationToken CancelToken { get; private set; } = new CancellationToken();
+        public NodeRunner NodeRunner { get; private set; } = null!;
+
+        public abstract int RequiredNumberOfNodes { get; }
+        public abstract TimeSpan RunTestEvery { get; }
+        public abstract TestFailMode TestFailMode { get; }
+        public virtual int EthereumAccountIndex { get { return -1; } }
+        public virtual string CustomK8sNamespace { get { return string.Empty; } }
+
+        public string Name
+        {
+            get
+            {
+                return GetType().Name;
+            }
+        }
+
+        public ContentId? UploadFile(CodexAccess node, TestFile file)
+        {
+            using var fileStream = File.OpenRead(file.Filename);
+
+            var logMessage = $"Uploading file {file.Describe()}...";
+            var response = Stopwatch.Measure(Log, logMessage, () =>
+            {
+                return node.UploadFile(fileStream);
+            });
+
+            if (string.IsNullOrEmpty(response)) return null;
+            if (response.StartsWith(UploadFailedMessage)) return null;
+            
+            Log.Log($"Uploaded file. Received contentId: '{response}'.");
+            return new ContentId(response);
+        }
+
+        public TestFile DownloadFile(CodexAccess node, ContentId contentId, string fileLabel = "")
+        {
+            var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
+            var file = FileManager.CreateEmptyTestFile(fileLabel);
+            Stopwatch.Measure(Log, logMessage, () => DownloadToFile(node, contentId.Id, file));
+            Log.Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
+            return file;
+        }
+
+        public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null)
+        {
+            var nodeRunner = new NodeRunner(Nodes, Configuration, TimeSet, Log, Configuration.CodexDeployment.Metadata.KubeNamespace, EthereumAccountIndex);
+            return nodeRunner.DownloadLog(container, tailLines);
+        }
+
+        private void DownloadToFile(CodexAccess node, string contentId, TestFile file)
+        {
+            using var fileStream = File.OpenWrite(file.Filename);
+            try
+            {
+                using var downloadStream = node.DownloadFile(contentId);
+                downloadStream.CopyTo(fileStream);
+            }
+            catch
+            {
+                Log.Log($"Failed to download file '{contentId}'.");
+                throw;
+            }
+        }
+    }
+
+    public enum TestFailMode
+    {
+        StopAfterFirstFailure,
+        AlwaysRunAllMoments
+    }
+}

ContinuousTests/ContinuousTestRunner.cs

@@ -0,0 +1,65 @@
+using DistTestCore;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class ContinuousTestRunner
+    {
+        private readonly K8sFactory k8SFactory = new K8sFactory();
+        private readonly ConfigLoader configLoader = new ConfigLoader();
+        private readonly TestFactory testFactory = new TestFactory();
+        private readonly Configuration config;
+        private readonly StartupChecker startupChecker;
+        private readonly CancellationToken cancelToken;
+
+        public ContinuousTestRunner(string[] args, CancellationToken cancelToken)
+        {
+            config = configLoader.Load(args);
+            startupChecker = new StartupChecker(config, cancelToken);
+            this.cancelToken = cancelToken;
+        }
+
+        public void Run()
+        {
+            startupChecker.Check();
+
+            var taskFactory = new TaskFactory();
+            var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "Overview");
+            overviewLog.Log("Continuous tests starting...");
+            var allTests = testFactory.CreateTests();
+
+            ClearAllCustomNamespaces(allTests, overviewLog);
+
+            var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
+
+            foreach (var testLoop in testLoops)
+            {
+                if (cancelToken.IsCancellationRequested) break;
+
+                overviewLog.Log("Launching test-loop for " + testLoop.Name);
+                testLoop.Begin();
+                Thread.Sleep(TimeSpan.FromSeconds(5));
+            }
+
+            overviewLog.Log("Finished launching test-loops.");
+            cancelToken.WaitHandle.WaitOne();
+            overviewLog.Log("Cancelling all test-loops...");
+            taskFactory.WaitAll();
+            overviewLog.Log("All tasks cancelled.");
+        }
+
+        private void ClearAllCustomNamespaces(ContinuousTest[] allTests, FixtureLog log)
+        {
+            foreach (var test in allTests) ClearAllCustomNamespaces(test, log);
+        }
+
+        private void ClearAllCustomNamespaces(ContinuousTest test, FixtureLog log)
+        {
+            if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
+
+            log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
+            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log);
+            lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources();
+        }
+    }
+}

ContinuousTests/ContinuousTests.csproj

@@ -12,10 +12,10 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
-    <ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
-    <ProjectReference Include="..\CodexTests\CodexTests.csproj" />
+    <ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
+    <ProjectReference Include="..\Logging\Logging.csproj" />
   </ItemGroup>
 
 </Project>

ContinuousTests/K8sFactory.cs

@@ -0,0 +1,34 @@
+using DistTestCore.Codex;
+using DistTestCore;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class K8sFactory
+    {
+        public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log)
+        {
+            var kubeConfig = GetKubeConfig(kubeConfigFile);
+            var lifecycleConfig = new DistTestCore.Configuration
+            (
+                kubeConfigFile: kubeConfig,
+                logPath: logPath,
+                logDebug: false,
+                dataFilesPath: dataFilePath,
+                codexLogLevel: CodexLogLevel.Debug,
+                k8sNamespacePrefix: customNamespace
+            );
+
+            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, string.Empty);
+            DefaultContainerRecipe.TestsType = "continuous-tests";
+            DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
+            return lifecycle;
+        }
+
+        private static string? GetKubeConfig(string kubeConfigFile)
+        {
+            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
+            return kubeConfigFile;
+        }
+    }
+}

ContinuousTests/NodeRunner.cs

@@ -0,0 +1,113 @@
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using DistTestCore;
+using KubernetesWorkflow;
+using NUnit.Framework;
+using Logging;
+using Utils;
+using DistTestCore.Logs;
+
+namespace ContinuousTests
+{
+    public class NodeRunner
+    {
+        private readonly K8sFactory k8SFactory = new K8sFactory();
+        private readonly CodexAccess[] nodes;
+        private readonly Configuration config;
+        private readonly ITimeSet timeSet;
+        private readonly BaseLog log;
+        private readonly string customNamespace;
+        private readonly int ethereumAccountIndex;
+
+        public NodeRunner(CodexAccess[] nodes, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex)
+        {
+            this.nodes = nodes;
+            this.config = config;
+            this.timeSet = timeSet;
+            this.log = log;
+            this.customNamespace = customNamespace;
+            this.ethereumAccountIndex = ethereumAccountIndex;
+        }
+
+        public void RunNode(Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
+        {
+            RunNode(nodes.ToList().PickOneRandom(), operation, 0.TestTokens());
+        }
+
+        public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
+        {
+            RunNode(bootstrapNode, operation, 0.TestTokens());
+        }
+
+        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
+        {
+            var subFile = log.CreateSubfile();
+            var description = container.Name;
+            var handler = new LogDownloadHandler(container, description, subFile);
+
+            log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
+
+            var lifecycle = CreateTestLifecycle();
+            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
+            flow.DownloadContainerLog(container, handler, tailLines);
+
+            return new DownloadedLog(subFile, description);
+        }
+
+        public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
+        {
+            var lifecycle = CreateTestLifecycle();
+            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
+
+            try
+            {
+                var debugInfo = bootstrapNode.GetDebugInfo();
+                Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
+
+                var startupConfig = new StartupConfig();
+                startupConfig.NameOverride = "TransientNode";
+                var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Trace);
+                codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
+                codexStartConfig.MarketplaceConfig.AccountIndexOverride = ethereumAccountIndex;
+                codexStartConfig.BootstrapSpr = debugInfo.spr;
+                startupConfig.Add(codexStartConfig);
+                startupConfig.Add(config.CodexDeployment.GethStartResult);
+                var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
+
+                var account = config.CodexDeployment.GethStartResult.CompanionNode.Accounts[ethereumAccountIndex];
+
+                var marketplaceNetwork = config.CodexDeployment.GethStartResult.MarketplaceNetwork;
+                if (mintTestTokens.Amount > 0)
+                {
+                    var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
+                    var interaction = marketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
+                    interaction.MintTestTokens(new[] { account.Account }, mintTestTokens.Amount, tokenAddress);
+                }
+
+                var container = rc.Containers[0];
+                var address = lifecycle.Configuration.GetAddress(container);
+                var codexAccess = new CodexAccess(log, container, lifecycle.TimeSet, address);
+                var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess);
+
+                try
+                {
+                    operation(codexAccess, marketAccess, lifecycle);
+                }
+                catch
+                {
+                    lifecycle.DownloadLog(container);
+                    throw;
+                }
+            }
+            finally
+            {
+                flow.DeleteTestResources();
+            }
+        }
+
+        private TestLifecycle CreateTestLifecycle()
+        {
+            return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log);
+        }
+    }
+}

ContinuousTests/Program.cs

@@ -5,6 +5,7 @@ public class Program
     public static void Main(string[] args)
     {
         Console.WriteLine("Codex Continous-Test-Runner.");
+        Console.WriteLine("Running...");
 
         var runner = new ContinuousTestRunner(args, Cancellation.Cts.Token);
 
@@ -15,25 +16,18 @@ public class Program
 
             Cancellation.Cts.Cancel();
         };
-
-        try
-        {
-            runner.Run();
-        }
-        catch (Exception ex)
-        {
-            Console.WriteLine(ex.ToString());
-        }
+        
+        runner.Run();
         Console.WriteLine("Done.");
     }
-}
 
-public static class Cancellation
-{
-    static Cancellation()
+    public static class Cancellation
     {
-        Cts = new CancellationTokenSource();
-    }
+        static Cancellation()
+        {
+            Cts = new CancellationTokenSource();
+        }
 
-    public static CancellationTokenSource Cts { get; }
+        public static CancellationTokenSource Cts { get; } 
+    }
 }

ContinuousTests/SingleTestRun.cs

@@ -1,61 +1,55 @@
-using Logging;
+using DistTestCore.Codex;
+using DistTestCore;
+using Logging;
 using Utils;
+using KubernetesWorkflow;
 using NUnit.Framework.Internal;
 using System.Reflection;
-using CodexPlugin;
-using DistTestCore.Logs;
-using Core;
-using KubernetesWorkflow.Types;
-using TaskFactory = Utils.TaskFactory;
+using static Program;
 
 namespace ContinuousTests
 {
     public class SingleTestRun
     {
+        private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
         private readonly List<Exception> exceptions = new List<Exception>();
-        private readonly EntryPoint entryPoint;
         private readonly TaskFactory taskFactory;
         private readonly Configuration config;
-        private readonly ILog overviewLog;
-        private readonly StatusLog statusLog;
+        private readonly BaseLog overviewLog;
         private readonly TestHandle handle;
         private readonly CancellationToken cancelToken;
-        private readonly ICodexNode[] nodes;
+        private readonly CodexAccess[] nodes;
+        private readonly FileManager fileManager;
         private readonly FixtureLog fixtureLog;
         private readonly string testName;
+        private readonly string dataFolder;
         private static int failureCount = 0;
 
-        public SingleTestRun(EntryPointFactory entryPointFactory,
-            TaskFactory taskFactory, Configuration config, ILog overviewLog, StatusLog statusLog, TestHandle handle,
-            StartupChecker startupChecker, CancellationToken cancelToken, string deployId)
+        public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken)
         {
             this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
-            this.statusLog = statusLog;
             this.handle = handle;
             this.cancelToken = cancelToken;
             testName = handle.Test.GetType().Name;
-            fixtureLog = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, deployId, testName);
-            entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath,
-                config.CodexDeployment.Metadata.KubeNamespace, fixtureLog);
+            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
             ApplyLogReplacements(fixtureLog, startupChecker);
 
             nodes = CreateRandomNodes();
+            dataFolder = config.DataPath + "-" + Guid.NewGuid();
+            fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
         }
 
-        public void Run(EventWaitHandle runFinishedHandle, Action<bool> resultHandler)
+        public void Run(EventWaitHandle runFinishedHandle)
         {
             taskFactory.Run(() =>
             {
                 try
                 {
-                    RunTest(resultHandler);
-
-                    entryPoint.Decommission(
-                        deleteKubernetesResources: false, // This would delete the continuous test net.
-                        deleteTrackedFiles: true
-                    );
+                    RunTest();
+                    fileManager.DeleteAllTestFiles();
+                    Directory.Delete(dataFolder, true);
                     runFinishedHandle.Set();
                 }
                 catch (Exception ex)
@@ -63,81 +57,40 @@ namespace ContinuousTests
                     overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex);
                     Environment.Exit(-1);
                 }
-            }, nameof(SingleTestRun));
+            });
         }
 
-        private void RunTest(Action<bool> resultHandler)
+        private void RunTest()
         {
-            var testStart = DateTime.UtcNow;
-            TimeSpan duration = TimeSpan.Zero;
-
             try
             {
                 RunTestMoments();
-                duration = DateTime.UtcNow - testStart;
 
-                OverviewLog($" > Test passed. ({Time.FormatDuration(duration)})");
-                UpdateStatusLogPassed(testStart, duration);
-
-                if (!config.KeepPassedTestLogs)
-                {
-                    fixtureLog.Delete();
-                }
-
-                resultHandler(true);
+                if (!config.KeepPassedTestLogs) fixtureLog.Delete();
             }
             catch (Exception ex)
             {
                 fixtureLog.Error("Test run failed with exception: " + ex);
                 fixtureLog.MarkAsFailed();
-                UpdateStatusLogFailed(testStart, duration, ex.ToString());
-
-                DownloadContainerLogs(testStart);
 
                 failureCount++;
-                resultHandler(false);
                 if (config.StopOnFailure > 0)
                 {
                     OverviewLog($"Failures: {failureCount} / {config.StopOnFailure}");
                     if (failureCount >= config.StopOnFailure)
                     {
-                        OverviewLog($"Configured to stop after {config.StopOnFailure} failures.");
+                        OverviewLog($"Configured to stop after {config.StopOnFailure} failures. Downloading cluster logs...");
+                        DownloadClusterLogs();
+                        OverviewLog("Log download finished. Cancelling test runner...");
                         Cancellation.Cts.Cancel();
                     }
                 }
             }
         }
 
-        private void DownloadContainerLogs(DateTime testStart)
-        {
-            // The test failed just now. We can't expect the logs to be available in elastic-search immediately:
-            Thread.Sleep(TimeSpan.FromMinutes(1));
-
-            var effectiveStart = testStart.Subtract(TimeSpan.FromSeconds(30));
-            if (config.FullContainerLogs)
-            {
-                effectiveStart = config.CodexDeployment.Metadata.StartUtc.Subtract(TimeSpan.FromSeconds(30));
-            }
-
-            var effectiveEnd = DateTime.UtcNow;
-            var elasticSearchLogDownloader = new ElasticSearchLogDownloader(entryPoint.Tools, fixtureLog);
-
-            foreach (var node in nodes)
-            {
-                var container = node.Container;
-                var deploymentName = container.RunningContainers.StartResult.Deployment.Name;
-                var namespaceName = container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace;
-                var openingLine =
-                    $"{namespaceName} - {deploymentName} = {node.Container.Name} = {node.GetDebugInfo().id}";
-                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart,
-                    effectiveEnd, openingLine);
-            }
-        }
-
         private void ApplyLogReplacements(FixtureLog fixtureLog, StartupChecker startupChecker)
         {
-            foreach (var replacement in startupChecker.LogReplacements)
-                fixtureLog.AddStringReplace(replacement.From, replacement.To);
+            foreach (var replacement in startupChecker.LogReplacements) fixtureLog.AddStringReplace(replacement.From, replacement.To);
         }
 
         private void RunTestMoments()
@@ -145,8 +98,10 @@ namespace ContinuousTests
             var earliestMoment = handle.GetEarliestMoment();
 
             var t = earliestMoment;
-            while (!cancelToken.IsCancellationRequested)
+            while (true)
             {
+                cancelToken.ThrowIfCancellationRequested();
+
                 RunMoment(t);
 
                 if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
@@ -169,12 +124,10 @@ namespace ContinuousTests
                     {
                         ThrowFailTest();
                     }
-
+                    OverviewLog(" > Test passed.");
                     return;
                 }
             }
-
-            fixtureLog.Log("Test run has been cancelled.");
         }
 
         private void ThrowFailTest()
@@ -186,35 +139,17 @@ namespace ContinuousTests
             throw new Exception(exceptionsMessage);
         }
 
-        private void UpdateStatusLogFailed(DateTime testStart, TimeSpan duration, string error)
+        private void DownloadClusterLogs()
         {
-            statusLog.ConcludeTest("Failed", duration, CreateStatusLogData(testStart, error));
-        }
+            var k8sFactory = new K8sFactory();
+            var log = new NullLog();
+            log.FullFilename = Path.Combine(config.LogPath, "NODE");
+            var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), log);
 
-        private void UpdateStatusLogPassed(DateTime testStart, TimeSpan duration)
-        {
-            statusLog.ConcludeTest("Passed", duration, CreateStatusLogData(testStart, "OK"));
-        }
-
-        private Dictionary<string, string> CreateStatusLogData(DateTime testStart, string message)
-        {
-            var result = entryPoint.GetPluginMetadata();
-            result.Add("teststart", testStart.ToString("o"));
-            result.Add("testname", testName);
-            result.Add("message", message);
-            result.Add("involvedpods", string.Join(",", nodes.Select(n => n.GetName())));
-            result.Add("involvedpodnames", string.Join(",", nodes.Select(n => n.GetPodInfo().Name)));
-
-            var error = message.Split(Environment.NewLine).First();
-            if (error.Contains(":")) error = error.Substring(1 + error.LastIndexOf(":"));
-            result.Add("error", error);
-
-            var upload = nodes.Select(n => n.TransferSpeeds.GetUploadSpeed()).ToList()!.OptionalAverage();
-            var download = nodes.Select(n => n.TransferSpeeds.GetDownloadSpeed()).ToList()!.OptionalAverage();
-            if (upload != null) result.Add("avgupload", upload.ToString());
-            if (download != null) result.Add("avgdownload", download.ToString());
-
-            return result;
+            foreach (var container in config.CodexDeployment.CodexContainers)
+            {
+                lifecycle.DownloadLog(container);
+            }
         }
 
         private string GetCombinedExceptionsMessage(Exception[] exceptions)
@@ -233,7 +168,6 @@ namespace ContinuousTests
             {
                 return UnpackException(a.InnerExceptions.First());
             }
-
             if (exception is TargetInvocationException t)
             {
                 return UnpackException(t.InnerException!);
@@ -262,7 +196,7 @@ namespace ContinuousTests
         private void InitializeTest(string name)
         {
             Log($" > Running TestMoment '{name}'");
-            handle.Test.Initialize(nodes, fixtureLog, entryPoint.Tools.GetFileManager(), config, cancelToken);
+            handle.Test.Initialize(nodes, fixtureLog, fileManager, config, cancelToken);
         }
 
         private void DecommissionTest()
@@ -288,26 +222,31 @@ namespace ContinuousTests
             return $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
         }
 
-        private ICodexNode[] CreateRandomNodes()
+        private CodexAccess[] CreateRandomNodes()
         {
             var containers = SelectRandomContainers();
             fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
-            return entryPoint.CreateInterface().WrapCodexContainers(containers).ToArray();
+            return codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
         }
 
-        private RunningContainers[] SelectRandomContainers()
+        private RunningContainer[] SelectRandomContainers()
         {
             var number = handle.Test.RequiredNumberOfNodes;
-            var containers = config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToList();
-            if (number == -1) return containers.ToArray();
+            if (number == -1) return config.CodexDeployment.CodexContainers;
 
-            var result = new RunningContainers[number];
+            var containers = config.CodexDeployment.CodexContainers.ToList();
+            var result = new RunningContainer[number];
             for (var i = 0; i < number; i++)
             {
                 result[i] = containers.PickOneRandom();
             }
-
             return result;
         }
+
+        private DistTestCore.Configuration CreateFileManagerConfiguration()
+        {
+            return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
+                CodexLogLevel.Error, string.Empty);
+        }
     }
-}
+}

ContinuousTests/StartupChecker.cs

@@ -1,21 +1,18 @@
-using CodexPlugin;
-using Core;
-using DistTestCore.Logs;
+using DistTestCore.Codex;
+using DistTestCore;
 using Logging;
-using Newtonsoft.Json;
 
 namespace ContinuousTests
 {
     public class StartupChecker
     {
         private readonly TestFactory testFactory = new TestFactory();
-        private readonly EntryPoint entryPoint;
+        private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
         private readonly Configuration config;
         private readonly CancellationToken cancelToken;
 
-        public StartupChecker(EntryPoint entryPoint, Configuration config, CancellationToken cancelToken)
+        public StartupChecker(Configuration config, CancellationToken cancelToken)
         {
-            this.entryPoint = entryPoint;
             this.config = config;
             this.cancelToken = cancelToken;
             LogReplacements = new List<BaseLogStringReplacement>();
@@ -23,10 +20,8 @@ namespace ContinuousTests
 
         public void Check()
         {
-            var log = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, config.CodexDeployment.Id,
-                "StartupChecks");
+            var log = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "StartupChecks");
             log.Log("Starting continuous test run...");
-            IncludeDeploymentConfiguration(log);
             log.Log("Checking configuration...");
             PreflightCheck(config);
             log.Log("Contacting Codex nodes...");
@@ -36,30 +31,6 @@ namespace ContinuousTests
 
         public List<BaseLogStringReplacement> LogReplacements { get; }
 
-        private void IncludeDeploymentConfiguration(ILog log)
-        {
-            log.Log("");
-            var deployment = config.CodexDeployment;
-            var workflow = entryPoint.Tools.CreateWorkflow();
-            foreach (var instance in deployment.CodexInstances)
-            {
-                foreach (var container in instance.Containers.Containers)
-                {
-                    var podInfo = workflow.GetPodInfo(container);
-                    log.Log($"Codex environment variables for '{container.Name}':");
-                    log.Log(
-                        $"Namespace: {container.RunningContainers.StartResult.Cluster.Configuration.KubernetesNamespace} - " +
-                        $"Pod name: {podInfo.Name} - Deployment name: {instance.Containers.StartResult.Deployment.Name}");
-                    var codexVars = container.Recipe.EnvVars;
-                    foreach (var vars in codexVars) log.Log(vars.ToString());
-                    log.Log("");
-                }
-            }
-
-            log.Log($"Deployment metadata: {JsonConvert.SerializeObject(deployment.Metadata)}");
-            log.Log("");
-        }
-
         private void PreflightCheck(Configuration config)
         {
             var tests = testFactory.CreateTests();
@@ -67,7 +38,6 @@ namespace ContinuousTests
             {
                 throw new Exception("Unable to find any tests.");
             }
-
             foreach (var test in tests)
             {
                 cancelToken.ThrowIfCancellationRequested();
@@ -91,15 +61,13 @@ namespace ContinuousTests
 
         private void CheckCodexNodes(BaseLog log, Configuration config)
         {
-            var nodes = entryPoint.CreateInterface()
-                .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Containers).ToArray());
+            var nodes = codexNodeFactory.Create(config, config.CodexDeployment.CodexContainers, log, new DefaultTimeSet());
             var pass = true;
             foreach (var n in nodes)
             {
                 cancelToken.ThrowIfCancellationRequested();
 
-                var address = n.Container.GetAddress(log, CodexContainerRecipe.ApiPortTag);
-                log.Log($"Checking {n.Container.Name} @ '{address}'...");
+                log.Log($"Checking {n.Container.Name} @ '{n.Address.Host}:{n.Address.Port}'...");
 
                 if (EnsureOnline(log, n))
                 {
@@ -107,18 +75,17 @@ namespace ContinuousTests
                 }
                 else
                 {
-                    log.Error($"No response from '{address}'.");
+                    log.Error($"No response from '{n.Address.Host}'.");
                     pass = false;
                 }
             }
-
             if (!pass)
             {
                 throw new Exception("Not all codex nodes responded.");
             }
         }
 
-        private bool EnsureOnline(BaseLog log, ICodexNode n)
+        private bool EnsureOnline(BaseLog log, CodexAccess n)
         {
             try
             {
@@ -132,7 +99,6 @@ namespace ContinuousTests
             {
                 return false;
             }
-
             return true;
         }
 
@@ -141,9 +107,30 @@ namespace ContinuousTests
             var errors = new List<string>();
             CheckRequiredNumberOfNodes(tests, errors);
             CheckCustomNamespaceClashes(tests, errors);
+            CheckEthereumIndexClashes(tests, errors);
             return errors;
         }
 
+        private void CheckEthereumIndexClashes(ContinuousTest[] tests, List<string> errors)
+        {
+            var offLimits = config.CodexDeployment.CodexContainers.Length;
+            foreach (var test in tests)
+            {
+                if (test.EthereumAccountIndex != -1)
+                {
+                    if (test.EthereumAccountIndex <= offLimits)
+                    {
+                        errors.Add($"Test '{test.Name}' has selected 'EthereumAccountIndex' = {test.EthereumAccountIndex}. All accounts up to and including {offLimits} are being used by the targetted Codex net. Select a different 'EthereumAccountIndex'.");
+                    }
+                }
+            }
+
+            DuplicatesCheck(tests, errors,
+                considerCondition: t => t.EthereumAccountIndex != -1,
+                getValue: t => t.EthereumAccountIndex,
+                propertyName: nameof(ContinuousTest.EthereumAccountIndex));
+        }
+
         private void CheckCustomNamespaceClashes(ContinuousTest[] tests, List<string> errors)
         {
             DuplicatesCheck(tests, errors,
@@ -152,8 +139,7 @@ namespace ContinuousTests
                 propertyName: nameof(ContinuousTest.CustomK8sNamespace));
         }
 
-        private void DuplicatesCheck(ContinuousTest[] tests, List<string> errors,
-            Func<ContinuousTest, bool> considerCondition, Func<ContinuousTest, object> getValue, string propertyName)
+        private void DuplicatesCheck(ContinuousTest[] tests, List<string> errors, Func<ContinuousTest, bool> considerCondition, Func<ContinuousTest, object> getValue, string propertyName)
         {
             foreach (var test in tests)
             {
@@ -163,8 +149,7 @@ namespace ContinuousTests
                     if (duplicates.Any())
                     {
                         duplicates.Add(test);
-                        errors.Add(
-                            $"Tests '{string.Join(",", duplicates.Select(d => d.Name))}' have the same '{propertyName}'. These must be unique.");
+                        errors.Add($"Tests '{string.Join(",", duplicates.Select(d => d.Name))}' have the same '{propertyName}'. These must be unique.");
                         return;
                     }
                 }
@@ -179,16 +164,14 @@ namespace ContinuousTests
                 {
                     if (test.RequiredNumberOfNodes < 1)
                     {
-                        errors.Add(
-                            $"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Test must require > 0 nodes, or -1 to select all nodes.");
+                        errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Test must require > 0 nodes, or -1 to select all nodes.");
                     }
-                    else if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexInstances.Length)
+                    else if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
                     {
-                        errors.Add(
-                            $"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexInstances.Length}");
+                        errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
                     }
                 }
             }
         }
     }
-}
+}

ContinuousTests/TaskFactory.cs

@@ -1,27 +1,15 @@
-namespace Utils
+namespace ContinuousTests
 {
     public class TaskFactory
     {
         private readonly object taskLock = new();
         private readonly List<Task> activeTasks = new List<Task>();
 
-        public void Run(Action action, string name)
+        public void Run(Action action)
         {
             lock (taskLock)
             {
-                activeTasks.Add(Task.Run(() => CatchException(action, name)).ContinueWith(CleanupTask, null));
-            }
-        }
-
-        private void CatchException(Action action, string name)
-        {
-            try
-            {
-                action();
-            }
-            catch (Exception ex)
-            {
-                Console.WriteLine($"Exception in task '{name}': " + ex);
+                activeTasks.Add(Task.Run(action).ContinueWith(CleanupTask, null));
             }
         }
 

ContinuousTests/TestLoop.cs

@@ -0,0 +1,69 @@
+using Logging;
+
+namespace ContinuousTests
+{
+    public class TestLoop
+    {
+        private readonly TaskFactory taskFactory;
+        private readonly Configuration config;
+        private readonly BaseLog overviewLog;
+        private readonly Type testType;
+        private readonly TimeSpan runsEvery;
+        private readonly StartupChecker startupChecker;
+        private readonly CancellationToken cancelToken;
+        private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset);
+
+        public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken)
+        {
+            this.taskFactory = taskFactory;
+            this.config = config;
+            this.overviewLog = overviewLog;
+            this.testType = testType;
+            this.runsEvery = runsEvery;
+            this.startupChecker = startupChecker;
+            this.cancelToken = cancelToken;
+            Name = testType.Name;
+        }
+
+        public string Name { get; }
+
+        public void Begin()
+        {
+            taskFactory.Run(() =>
+            {
+                try
+                {
+                    while (true)
+                    {
+                        WaitHandle.WaitAny(new[] { runFinishedHandle, cancelToken.WaitHandle });
+
+                        cancelToken.ThrowIfCancellationRequested();
+
+                        StartTest();
+
+                        cancelToken.WaitHandle.WaitOne(runsEvery);
+                    }
+                }
+                catch (OperationCanceledException)
+                {
+                    overviewLog.Log("Test-loop " + testType.Name + " is cancelled.");
+                }
+                catch (Exception ex)
+                {
+                    overviewLog.Error("Test infra failure: TestLoop failed with " + ex);
+                    Environment.Exit(-1);
+                }
+            });
+        }
+
+        private void StartTest()
+        {
+            var test = (ContinuousTest)Activator.CreateInstance(testType)!;
+            var handle = new TestHandle(test);
+            var run = new SingleTestRun(taskFactory, config, overviewLog, handle, startupChecker, cancelToken);
+
+            runFinishedHandle.Reset();
+            run.Run(runFinishedHandle);
+        }
+    }
+}

ContinuousTests/Tests/FetchTest.cs

@@ -0,0 +1,18 @@
+using DistTestCore.Helpers;
+
+namespace ContinuousTests.Tests
+{
+    public class FetchTest : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => -1;
+        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);
+        public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
+
+        [TestMoment(t: 0)]
+        public void CheckConnectivity()
+        {
+            var checker = new PeerFetchTestHelpers(Log, FileManager);
+            checker.AssertFullFetchInterconnectivity(Nodes);
+        }
+    }
+}

ContinuousTests/Tests/HoldMyBeerTest.cs

@@ -1,7 +1,5 @@
-using CodexPlugin;
-using FileUtils;
+using DistTestCore;
 using NUnit.Framework;
-using Utils;
 
 namespace ContinuousTests.Tests
 {
@@ -12,19 +10,19 @@ namespace ContinuousTests.Tests
         public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
 
         private ContentId? cid;
-        private TrackedFile file = null!;
+        private TestFile file = null!;
 
         [TestMoment(t: Zero)]
         public void UploadTestFile()
         {
             var filesize = 80.MB();
 
-            file = FileManager.GenerateFile(filesize);
+            file = FileManager.GenerateTestFile(filesize);
 
-            cid = Nodes[0].UploadFile(file);
+            cid = UploadFile(Nodes[0], file);
             Assert.That(cid, Is.Not.Null);
 
-            var dl = Nodes[0].DownloadContent(cid);
+            var dl = DownloadFile(Nodes[0], cid!);
             file.AssertIsEqual(dl);
         }
     }

ContinuousTests/Tests/PeersTest.cs

@@ -1,9 +1,8 @@
-using CodexPlugin;
-using CodexTests.Helpers;
-using ContinuousTests;
+using DistTestCore.Codex;
+using DistTestCore.Helpers;
 using NUnit.Framework;
 
-namespace CodexContinuousTests.Tests
+namespace ContinuousTests.Tests
 {
     public class PeersTest : ContinuousTest
     {
@@ -38,7 +37,7 @@ namespace CodexContinuousTests.Tests
             }
         }
 
-        private string AreAllPresent(ICodexNode n, string[] allIds)
+        private string AreAllPresent(CodexAccess n, string[] allIds)
         {
             var info = n.GetDebugInfo();
             var known = info.table.nodes.Select(n => n.nodeId).ToArray();

ContinuousTests/Tests/ThresholdChecks.cs

@@ -0,0 +1,60 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class ThresholdChecks : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => 1;
+        public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+
+        private static readonly List<string> previousBreaches = new List<string>();
+
+        [TestMoment(t: 0)]
+        public void CheckAllThresholds()
+        {
+            var allNodes = CreateAccessToAllNodes();
+            foreach (var n in allNodes) CheckThresholds(n);
+        }
+
+        private void CheckThresholds(CodexAccess n)
+        {
+            var breaches = n.GetDebugThresholdBreaches();
+            if (breaches.breaches.Any())
+            {
+                var newBreaches = new List<string>();
+                foreach (var b in breaches.breaches)
+                {
+                    if (!previousBreaches.Contains(b))
+                    {
+                        newBreaches.Add(b);
+                        previousBreaches.Add(b);
+                    }
+                }
+
+                if (newBreaches.Any())
+                {
+                    Assert.Fail(string.Join(",", newBreaches.Select(b => FormatBreach(n, b))));
+
+                    Program.Cancellation.Cts.Cancel();
+                }
+            }
+        }
+
+        private string FormatBreach(CodexAccess n, string breach)
+        {
+            return $"{n.Container.Name} = '{breach}'";
+        }
+
+        private CodexAccess[] CreateAccessToAllNodes()
+        {
+            // Normally, a continuous test accesses only a subset of the nodes in the deployment.
+            // This time, we want to check all of them.
+            var factory = new CodexAccessFactory();
+            var allContainers = Configuration.CodexDeployment.CodexContainers;
+            return factory.Create(Configuration, allContainers, Log, new DefaultTimeSet());
+        }
+    }
+}

ContinuousTests/Tests/TwoClientTest.cs

@@ -0,0 +1,32 @@
+using DistTestCore;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class TwoClientTest : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => 2;
+        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+
+        private ContentId? cid;
+        private TestFile file = null!;
+
+        [TestMoment(t: Zero)]
+        public void UploadTestFile()
+        {
+            file = FileManager.GenerateTestFile(80.MB());
+
+            cid = UploadFile(Nodes[0], file);
+            Assert.That(cid, Is.Not.Null);
+        }
+
+        [TestMoment(t: 10)]
+        public void DownloadTestFile()
+        {
+            var dl = DownloadFile(Nodes[1], cid!);
+
+            file.AssertIsEqual(dl);
+        }
+    }
+}

ContinuousTests/run.sh

@@ -0,0 +1,6 @@
+dotnet run \
+    --kube-config=/opt/kubeconfig.yaml \
+    --codex-deployment=codex-deployment.json \
+    --keep=1 \
+    --stop=10 \
+    --dl-logs=1

DistTestCore/AutoBootstrapDistTest.cs

@@ -0,0 +1,30 @@
+using NUnit.Framework;
+
+namespace DistTestCore
+{
+    public class AutoBootstrapDistTest : DistTest
+    {
+        public override IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
+        {
+            throw new Exception("AutoBootstrapDistTest creates and attaches a single bootstrap node for you. " +
+                "If you want to control the bootstrap node from your test, please use DistTest instead.");
+        }
+
+        public override ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        {
+            var codexSetup = CreateCodexSetup(numberOfNodes);
+            setup(codexSetup);
+            codexSetup.WithBootstrapNode(BootstrapNode);
+            return BringOnline(codexSetup);
+        }
+
+        [SetUp]
+        public void SetUpBootstrapNode()
+        {
+            var setup = CreateCodexSetup(1).WithName("BOOTSTRAP");
+            BootstrapNode = BringOnline(setup)[0];
+        }
+
+        protected IOnlineCodexNode BootstrapNode { get; private set; } = null!;
+    }
+}

DistTestCore/BaseStarter.cs

@@ -0,0 +1,42 @@
+using Logging;
+
+namespace DistTestCore
+{
+    public class BaseStarter
+    {
+        protected readonly TestLifecycle lifecycle;
+        private Stopwatch? stopwatch;
+
+        public BaseStarter(TestLifecycle lifecycle)
+        {
+            this.lifecycle = lifecycle;
+        }
+
+        protected void LogStart(string msg)
+        {
+            Log(msg);
+            stopwatch = Stopwatch.Begin(lifecycle.Log, GetClassName());
+        }
+
+        protected void LogEnd(string msg)
+        {
+            stopwatch!.End(msg);
+            stopwatch = null;
+        }
+
+        protected void Log(string msg)
+        {
+            lifecycle.Log.Log($"{GetClassName()} {msg}");
+        }
+
+        protected void Debug(string msg)
+        {
+            lifecycle.Log.Debug($"{GetClassName()} {msg}", 1);
+        }
+
+        private string GetClassName()
+        {
+            return $"({GetType().Name})";
+        }
+    }
+}

DistTestCore/ByteSize.cs

@@ -1,7 +1,10 @@
-namespace Utils
+using Utils;
+
+namespace DistTestCore
 {
     public class ByteSize
     {
+
         public ByteSize(long sizeInBytes)
         {
             if (sizeInBytes < 0) throw new ArgumentException("Cannot create ByteSize object with size less than 0. Was: " + sizeInBytes);
@@ -10,20 +13,11 @@
 
         public long SizeInBytes { get; }
 
-        public const double DefaultSecondsPerMB = 10.0;
-
         public long ToMB()
         {
             return SizeInBytes / (1024 * 1024);
         }
 
-        public ByteSize Multiply(double factor)
-        {
-            double bytes = SizeInBytes;
-            double result = Math.Round(bytes * factor);
-            return new ByteSize(Convert.ToInt64(result));
-        }
-
         public override bool Equals(object? obj)
         {
             return obj is ByteSize size && SizeInBytes == size.SizeInBytes;
@@ -38,24 +32,6 @@
         {
             return Formatter.FormatByteSize(SizeInBytes);
         }
-
-        public TimeSpan ToTimeSpan(double secsPerMB = DefaultSecondsPerMB)
-        {
-            var filesizeInMb = SizeInBytes / (1024 * 1024);
-            return TimeSpan.FromSeconds(filesizeInMb * secsPerMB);
-        }
-    }
-
-    public class BytesPerSecond : ByteSize
-    {
-        public BytesPerSecond(long sizeInBytes) : base(sizeInBytes)
-        {
-        }
-
-        public override string ToString()
-        {
-            return base.ToString() + "/s";
-        }
     }
 
     public static class ByteSizeIntExtensions
@@ -111,6 +87,5 @@
         {
             return Convert.ToInt64(i).TB();
         }
-
     }
 }

DistTestCore/Codex/CodexAccess.cs

@@ -1,27 +1,28 @@
-using Core;
-using KubernetesWorkflow;
-using KubernetesWorkflow.Types;
+using KubernetesWorkflow;
+using Logging;
 using Utils;
 
-namespace CodexPlugin
+namespace DistTestCore.Codex
 {
     public class CodexAccess : ILogHandler
     {
-        private readonly IPluginTools tools;
+        private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
         private bool hasContainerCrashed;
 
-        public CodexAccess(IPluginTools tools, RunningContainer container, CrashWatcher crashWatcher)
+        public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
         {
-            this.tools = tools;
+            this.log = log;
             Container = container;
-            CrashWatcher = crashWatcher;
+            this.timeSet = timeSet;
+            Address = address;
             hasContainerCrashed = false;
 
-            CrashWatcher.Start(this);
+            if (container.CrashWatcher != null) container.CrashWatcher.Start(this);
         }
 
         public RunningContainer Container { get; }
-        public CrashWatcher CrashWatcher { get; }
+        public Address Address { get; }
 
         public CodexDebugResponse GetDebugInfo()
         {
@@ -41,19 +42,15 @@ namespace CodexPlugin
                 };
             }
 
-            var result = http.Deserialize<CodexDebugPeerResponse>(str);
+            var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
             result.IsPeerFound = true;
             return result;
         }
 
-        public CodexDebugBlockExchangeResponse GetDebugBlockExchange()
+        public CodexDebugFetchResponse DebugFetch(string contentId)
         {
-            return Http().HttpGetJson<CodexDebugBlockExchangeResponse>("debug/blockexchange");
-        }
-
-        public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
-        {
-            return LongHttp().HttpGetJson<CodexDebugRepoStoreResponse[]>("debug/repostore");
+            var http = Http();
+            return http.HttpGetJson<CodexDebugFetchResponse>($"debug/fetch/{contentId}");
         }
 
         public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
@@ -63,17 +60,12 @@ namespace CodexPlugin
 
         public string UploadFile(FileStream fileStream)
         {
-            return Http().HttpPostStream("data", fileStream);
+            return Http().HttpPostStream("upload", fileStream);
         }
 
         public Stream DownloadFile(string contentId)
         {
-            return Http().HttpGetStream("data/" + contentId + "/network");
-        }
-
-        public CodexLocalDataResponse[] LocalFiles()
-        {
-            return Http().HttpGetJson<CodexLocalDataResponse[]>("data");
+            return Http().HttpGetStream("download/" + contentId);
         }
 
         public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
@@ -101,25 +93,9 @@ namespace CodexPlugin
             return Container.Name;
         }
 
-        public PodInfo GetPodInfo()
+        private Http Http()
         {
-            var workflow = tools.CreateWorkflow();
-            return workflow.GetPodInfo(Container);
-        }
-
-        private IHttp Http()
-        {
-            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
-        }
-
-        private IHttp LongHttp()
-        {
-            return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name);
-        }
-
-        private Address GetAddress()
-        {
-            return Container.GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
+            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
         }
 
         private void CheckContainerCrashed(HttpClient client)
@@ -129,10 +105,8 @@ namespace CodexPlugin
 
         public void Log(Stream crashLog)
         {
-            var log = tools.GetLog();
             var file = log.CreateSubfile();
             log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
-            file.Write($"Container Crash Log for {Container.Name}.");
 
             using var reader = new StreamReader(crashLog);
             var line = reader.ReadLine();

DistTestCore/Codex/CodexApiTypes.cs

@@ -1,14 +1,13 @@
 using Newtonsoft.Json;
 
-namespace CodexPlugin
+namespace DistTestCore.Codex
 {
     public class CodexDebugResponse
     {
         public string id { get; set; } = string.Empty;
-        public string[] addrs { get; set; } = Array.Empty<string>();
+        public string[] addrs { get; set; } = new string[0];
         public string repo { get; set; } = string.Empty;
         public string spr { get; set; } = string.Empty;
-        public string[] announceAddresses { get; set; } = Array.Empty<string>();
         public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
         public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
         public CodexDebugVersionResponse codex { get; set; } = new();
@@ -77,7 +76,29 @@ namespace CodexPlugin
 
         public string peerId { get; set; } = string.Empty;
         public long seqNo { get; set; }
-        public string[] addresses { get; set; } = Array.Empty<string>();
+        public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
+    }
+
+    public class CodexDebugPeerAddressResponse
+    {
+        public string address { get; set; } = string.Empty;
+    }
+
+    public class CodexDebugFetchResponse
+    {
+        public string originalBytes { get; set; } = string.Empty;
+        public string blockSize { get; set; } = string.Empty;
+        public string numberOfBlocks { get; set; } = string.Empty;
+        public string version { get; set; } = string.Empty;
+        public string hcodec { get; set; } = string.Empty;
+        public string codec { get; set; } = string.Empty;
+        public string @protected { get; set; } = string.Empty;
+        public CodexDebugFetchBlockResponse[] blocks { get; set; } = Array.Empty<CodexDebugFetchBlockResponse>();
+    }
+
+    public class CodexDebugFetchBlockResponse
+    {
+        public string cid { get; set; } = string.Empty;
     }
 
     public class CodexDebugThresholdBreaches
@@ -118,78 +139,4 @@ namespace CodexPlugin
         public string state { get; set; } = string.Empty;
         public string error { get; set; } = string.Empty;
     }
-
-    public class CodexDebugBlockExchangeResponse
-    {
-        public CodexDebugBlockExchangeResponsePeer[] peers { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeer>();
-        public int taskQueue { get; set; }
-        public int pendingBlocks { get; set; }
-
-        public override string ToString()
-        {
-            if (peers.Length == 0 && taskQueue == 0 && pendingBlocks == 0) return "all-empty";
-
-            return $"taskqueue: {taskQueue} pendingblocks: {pendingBlocks} peers: {string.Join(",", peers.Select(p => p.ToString()))}";
-        }
-    }
-
-    public class CodexDebugBlockExchangeResponsePeer
-    {
-        public string peerid { get; set; } = string.Empty;
-        public CodexDebugBlockExchangeResponsePeerHasBlock[] hasBlocks { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeerHasBlock>();
-        public CodexDebugBlockExchangeResponsePeerWant[] wants { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeerWant>();
-        public int exchanged { get; set; }
-
-        public override string ToString()
-        {
-            return $"(blocks:{hasBlocks.Length} wants:{wants.Length})";
-        }
-    }
-
-    public class CodexDebugBlockExchangeResponsePeerHasBlock
-    {
-        public string cid { get; set; } = string.Empty;
-        public bool have { get; set; }
-        public string price { get; set; } = string.Empty;
-    }
-
-    public class CodexDebugBlockExchangeResponsePeerWant
-    {
-        public string block { get; set; } = string.Empty;
-        public int priority { get; set; }
-        public bool cancel { get; set; }
-        public string wantType { get; set; } = string.Empty;
-        public bool sendDontHave { get; set; }
-    }
-
-    public class CodexDebugRepoStoreResponse
-    {
-        public string cid { get; set; } = string.Empty; 
-    }
-
-    public class CodexLocalData
-    {
-        public CodexLocalData(ContentId cid, CodexLocalDataManifestResponse manifest)
-        {
-            Cid = cid;
-            Manifest = manifest;
-        }
-
-        public ContentId Cid { get; }
-        public CodexLocalDataManifestResponse Manifest { get; }
-    }
-
-    public class CodexLocalDataResponse
-    {
-        public string cid { get; set; } = string.Empty;
-        public CodexLocalDataManifestResponse manifest { get; set; } = new();
-    }
-
-    public class CodexLocalDataManifestResponse
-    {
-        public string rootHash { get; set; } = string.Empty;
-        public int originalBytes { get; set; }
-        public int blockSize { get; set; }
-        public bool @protected { get; set; }
-    }
 }

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -0,0 +1,108 @@
+using DistTestCore.Marketplace;
+using KubernetesWorkflow;
+
+namespace DistTestCore.Codex
+{
+    public class CodexContainerRecipe : DefaultContainerRecipe
+    {
+        //private const string DefaultDockerImage = "codexstorage/nim-codex:sha-d279eeb-dist-tests";
+        private const string DefaultDockerImage = "thatbenbierens/nim-codex:debugfetch";
+
+        public const string MetricsPortTag = "metrics_port";
+        public const string DiscoveryPortTag = "discovery-port";
+
+        // Used by tests for time-constraint assertions.
+        public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
+        public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
+
+        public override string AppName => "codex";
+        public override string Image { get; }
+
+        public CodexContainerRecipe()
+        {
+            Image = GetDockerImage();
+        }
+
+        protected override void InitializeRecipe(StartupConfig startupConfig)
+        {
+            var config = startupConfig.Get<CodexStartupConfig>();
+
+            AddExposedPortAndVar("CODEX_API_PORT");
+            AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
+
+            AddEnvVar("CODEX_DATA_DIR", $"datadir{ContainerNumber}");
+            AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
+            AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
+
+            // This makes the node announce itself to its local (pod) IP address.
+            AddEnvVar("NAT_IP_AUTO", "true");
+
+            var listenPort = AddInternalPort();
+            AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
+
+            if (!string.IsNullOrEmpty(config.BootstrapSpr))
+            {
+                AddEnvVar("CODEX_BOOTSTRAP_NODE", config.BootstrapSpr);
+            }
+            if (config.StorageQuota != null)
+            {
+                AddEnvVar("CODEX_STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
+            }
+            if (config.BlockTTL != null)
+            {
+                AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
+            }
+            if (config.BlockMaintenanceInterval != null)
+            {
+                AddEnvVar("CODEX_BLOCK_MI", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
+            }
+            if (config.BlockMaintenanceNumber != null)
+            {
+                AddEnvVar("CODEX_BLOCK_MN", config.BlockMaintenanceNumber.ToString()!);
+            }
+            if (config.MetricsMode != Metrics.MetricsMode.None)
+            {
+                var metricsPort = AddInternalPort(MetricsPortTag);
+                AddEnvVar("CODEX_METRICS", "true");
+                AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
+                AddEnvVar("CODEX_METRICS_PORT", metricsPort);
+                AddPodAnnotation("prometheus.io/scrape", "true");
+                AddPodAnnotation("prometheus.io/port", metricsPort.Number.ToString());
+            }
+
+            if (config.MarketplaceConfig != null)
+            {
+                var gethConfig = startupConfig.Get<GethStartResult>();
+                var companionNode = gethConfig.CompanionNode;
+                var companionNodeAccount = companionNode.Accounts[GetAccountIndex(config.MarketplaceConfig)];
+                Additional(companionNodeAccount);
+
+                var ip = companionNode.RunningContainer.Pod.PodInfo.Ip;
+                var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number;
+
+                AddEnvVar("CODEX_ETH_PROVIDER", $"ws://{ip}:{port}");
+                AddEnvVar("CODEX_ETH_ACCOUNT", companionNodeAccount.Account);
+                AddEnvVar("CODEX_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
+                AddEnvVar("CODEX_PERSISTENCE", "true");
+
+                if (config.MarketplaceConfig.IsValidator)
+                {
+                    AddEnvVar("CODEX_VALIDATOR", "true");
+                }
+            }
+        }
+
+        private int GetAccountIndex(MarketplaceInitialConfig marketplaceConfig)
+        {
+            if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value;
+            return Index;
+        }
+
+        private string GetDockerImage()
+        {
+            var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
+            if (!string.IsNullOrEmpty(image)) return image;
+            return DefaultDockerImage;
+        }
+    }
+}

DistTestCore/Codex/CodexDeployment.cs

@@ -0,0 +1,57 @@
+using DistTestCore.Marketplace;
+using KubernetesWorkflow;
+
+namespace DistTestCore.Codex
+{
+    public class CodexDeployment
+    {
+        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, GrafanaStartInfo? grafanaStartInfo, DeploymentMetadata metadata)
+        {
+            GethStartResult = gethStartResult;
+            CodexContainers = codexContainers;
+            PrometheusContainer = prometheusContainer;
+            GrafanaStartInfo = grafanaStartInfo;
+            Metadata = metadata;
+        }
+
+        public GethStartResult GethStartResult { get; }
+        public RunningContainer[] CodexContainers { get; }
+        public RunningContainer? PrometheusContainer { get; }
+        public GrafanaStartInfo? GrafanaStartInfo { get; }
+        public DeploymentMetadata Metadata { get; }
+    }
+
+    public class DeploymentMetadata
+    {
+        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
+        {
+            DeployDateTimeUtc = DateTime.UtcNow;
+            KubeNamespace = kubeNamespace;
+            NumberOfCodexNodes = numberOfCodexNodes;
+            NumberOfValidators = numberOfValidators;
+            StorageQuotaMB = storageQuotaMB;
+            CodexLogLevel = codexLogLevel;
+            InitialTestTokens = initialTestTokens;
+            MinPrice = minPrice;
+            MaxCollateral = maxCollateral;
+            MaxDuration = maxDuration;
+            BlockTTL = blockTTL;
+            BlockMI = blockMI;
+            BlockMN = blockMN;
+        }
+
+        public DateTime DeployDateTimeUtc { get; }
+        public string KubeNamespace { get; }
+        public int NumberOfCodexNodes { get; }
+        public int NumberOfValidators { get; }
+        public int StorageQuotaMB { get; }
+        public CodexLogLevel CodexLogLevel { get; }
+        public int InitialTestTokens { get; }
+        public int MinPrice { get; }
+        public int MaxCollateral { get; }
+        public int MaxDuration { get; }
+        public int BlockTTL { get; }
+        public int BlockMI { get; }
+        public int BlockMN { get; }
+    }
+}

DistTestCore/Codex/CodexLogLevel.cs

@@ -1,4 +1,4 @@
-namespace CodexPlugin
+namespace DistTestCore.Codex
 {
     public enum CodexLogLevel
     {

DistTestCore/Codex/CodexStartupConfig.cs

@@ -0,0 +1,25 @@
+using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
+using KubernetesWorkflow;
+
+namespace DistTestCore.Codex
+{
+    public class CodexStartupConfig
+    {
+        public CodexStartupConfig(CodexLogLevel logLevel)
+        {
+            LogLevel = logLevel;
+        }
+
+        public string? NameOverride { get; set; }
+        public Location Location { get; set; }
+        public CodexLogLevel LogLevel { get; }
+        public ByteSize? StorageQuota { get; set; }
+        public MetricsMode MetricsMode { get; set; }
+        public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
+        public string? BootstrapSpr { get; set; }
+        public int? BlockTTL { get; set; }
+        public TimeSpan? BlockMaintenanceInterval { get; set; }
+        public int? BlockMaintenanceNumber { get; set; }
+    }
+}

DistTestCore/CodexNodeFactory.cs

@@ -0,0 +1,32 @@
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
+
+namespace DistTestCore
+{
+    public interface ICodexNodeFactory
+    {
+        OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group);
+    }
+
+    public class CodexNodeFactory : ICodexNodeFactory
+    {
+        private readonly TestLifecycle lifecycle;
+        private readonly IMetricsAccessFactory metricsAccessFactory;
+        private readonly IMarketplaceAccessFactory marketplaceAccessFactory;
+
+        public CodexNodeFactory(TestLifecycle lifecycle, IMetricsAccessFactory metricsAccessFactory, IMarketplaceAccessFactory marketplaceAccessFactory)
+        {
+            this.lifecycle = lifecycle;
+            this.metricsAccessFactory = metricsAccessFactory;
+            this.marketplaceAccessFactory = marketplaceAccessFactory;
+        }
+
+        public OnlineCodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group)
+        {
+            var metricsAccess = metricsAccessFactory.CreateMetricsAccess(access.Container);
+            var marketplaceAccess = marketplaceAccessFactory.CreateMarketplaceAccess(access);
+            return new OnlineCodexNode(lifecycle, access, group, metricsAccess, marketplaceAccess);
+        }
+    }
+}

DistTestCore/CodexNodeGroup.cs

@@ -1,29 +1,29 @@
-using Core;
-using KubernetesWorkflow.Types;
-using MetricsPlugin;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
 using System.Collections;
 
-namespace CodexPlugin
+namespace DistTestCore
 {
-    public interface ICodexNodeGroup : IEnumerable<ICodexNode>, IHasManyMetricScrapeTargets
+    public interface ICodexNodeGroup : IEnumerable<IOnlineCodexNode>
     {
-        void BringOffline(bool waitTillStopped);
-        ICodexNode this[int index] { get; }
+        ICodexSetup BringOffline();
+        IOnlineCodexNode this[int index] { get; }
     }
 
     public class CodexNodeGroup : ICodexNodeGroup
     {
-        private readonly CodexStarter starter;
+        private readonly TestLifecycle lifecycle;
 
-        public CodexNodeGroup(CodexStarter starter, IPluginTools tools, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
         {
-            this.starter = starter;
+            this.lifecycle = lifecycle;
+            Setup = setup;
             Containers = containers;
-            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray();
+            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
             Version = new CodexDebugVersionResponse();
         }
 
-        public ICodexNode this[int index]
+        public IOnlineCodexNode this[int index]
         {
             get
             {
@@ -31,22 +31,27 @@ namespace CodexPlugin
             }
         }
 
-        public void BringOffline(bool waitTillStopped)
+        public ICodexSetup BringOffline()
         {
-            starter.BringOffline(this, waitTillStopped);
+            lifecycle.CodexStarter.BringOffline(this);
+
+            var result = Setup;
             // Clear everything. Prevent accidental use.
-            Nodes = Array.Empty<CodexNode>();
+            Setup = null!;
+            Nodes = Array.Empty<OnlineCodexNode>();
             Containers = null!;
+
+            return result;
         }
 
+        public CodexSetup Setup { get; private set; }
         public RunningContainers[] Containers { get; private set; }
-        public CodexNode[] Nodes { get; private set; }
+        public OnlineCodexNode[] Nodes { get; private set; }
         public CodexDebugVersionResponse Version { get; private set; }
-        public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray();
 
-        public IEnumerator<ICodexNode> GetEnumerator()
+        public IEnumerator<IOnlineCodexNode> GetEnumerator()
         {
-            return Nodes.Cast<ICodexNode>().GetEnumerator();
+            return Nodes.Cast<IOnlineCodexNode>().GetEnumerator();
         }
 
         IEnumerator IEnumerable.GetEnumerator()
@@ -74,10 +79,9 @@ namespace CodexPlugin
             Version = first;
         }
 
-        private CodexNode CreateOnlineCodexNode(RunningContainer c, IPluginTools tools, ICodexNodeFactory factory)
+        private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
         {
-            var watcher = factory.CreateCrashWatcher(c);
-            var access = new CodexAccess(tools, c, watcher);
+            var access = new CodexAccess(lifecycle.Log, c, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(c));
             return factory.CreateOnlineCodexNode(access, this);
         }
     }

DistTestCore/CodexSetup.cs

@@ -0,0 +1,109 @@
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using KubernetesWorkflow;
+
+namespace DistTestCore
+{
+    public interface ICodexSetup
+    {
+        ICodexSetup WithName(string name);
+        ICodexSetup At(Location location);
+        ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
+        ICodexSetup WithStorageQuota(ByteSize storageQuota);
+        ICodexSetup WithBlockTTL(TimeSpan duration);
+        ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration);
+        ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks);
+        ICodexSetup EnableMetrics();
+        ICodexSetup EnableMarketplace(TestToken initialBalance);
+        ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
+        ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator);
+    }
+    
+    public class CodexSetup : CodexStartupConfig, ICodexSetup
+    {
+        public int NumberOfNodes { get; }
+
+        public CodexSetup(int numberOfNodes, CodexLogLevel logLevel)
+            : base(logLevel)
+        {
+            NumberOfNodes = numberOfNodes;
+        }
+
+        public ICodexSetup WithName(string name)
+        {
+            NameOverride = name;
+            return this;
+        }
+
+        public ICodexSetup At(Location location)
+        {
+            Location = location;
+            return this;
+        }
+
+        public ICodexSetup WithBootstrapNode(IOnlineCodexNode node)
+        {
+            BootstrapSpr = node.GetDebugInfo().spr;
+            return this;
+        }
+
+        public ICodexSetup WithStorageQuota(ByteSize storageQuota)
+        {
+            StorageQuota = storageQuota;
+            return this;
+        }
+
+        public ICodexSetup WithBlockTTL(TimeSpan duration)
+        {
+            BlockTTL = Convert.ToInt32(duration.TotalSeconds);
+            return this;
+        }
+
+        public ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration)
+        {
+            BlockMaintenanceInterval = duration;
+            return this;
+        }
+
+        public ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks)
+        {
+            BlockMaintenanceNumber = numberOfBlocks;
+            return this;
+        }
+
+        public ICodexSetup EnableMetrics()
+        {
+            MetricsMode = Metrics.MetricsMode.Record;
+            return this;
+        }
+
+        public ICodexSetup EnableMarketplace(TestToken initialBalance)
+        {
+            return EnableMarketplace(initialBalance, 1000.Eth());
+        }
+
+        public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther)
+        {
+            return EnableMarketplace(initialBalance, initialEther, false);
+        }
+
+        public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator)
+        {
+            MarketplaceConfig = new MarketplaceInitialConfig(initialEther, initialBalance, isValidator);
+            return this;
+        }
+
+        public string Describe()
+        {
+            var args = string.Join(',', DescribeArgs());
+            return $"({NumberOfNodes} CodexNodes with args:[{args}])";
+        }
+
+        private IEnumerable<string> DescribeArgs()
+        {
+            yield return $"LogLevel={LogLevel}";
+            if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
+            if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
+        }
+    }
+}

DistTestCore/CodexStarter.cs

@@ -0,0 +1,158 @@
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
+using KubernetesWorkflow;
+using Logging;
+
+namespace DistTestCore
+{
+    public class CodexStarter : BaseStarter
+    {
+        public CodexStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
+        {
+        }
+
+        public List<CodexNodeGroup> RunningGroups { get; } = new List<CodexNodeGroup>();
+
+        public ICodexNodeGroup BringOnline(CodexSetup codexSetup)
+        {
+            LogSeparator();
+            LogStart($"Starting {codexSetup.Describe()}...");
+            var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
+
+            var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
+
+            var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
+
+            var metricAccessFactory = CollectMetrics(codexSetup, containers);
+            
+            var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
+
+            var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
+            lifecycle.SetCodexVersion(group.Version);
+
+            var nl = Environment.NewLine;
+            var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
+            LogEnd($"Started {codexSetup.NumberOfNodes} nodes " +
+                $"of image '{containers.Containers().First().Recipe.Image}' " +
+                $"and version '{group.Version}'{nl}" +
+                podInfos);
+            LogSeparator();
+
+            return group;
+        }
+
+        public void BringOffline(CodexNodeGroup group)
+        {
+            LogStart($"Stopping {group.Describe()}...");
+            var workflow = CreateWorkflow();
+            foreach (var c in group.Containers)
+            {
+                StopCrashWatcher(c);
+                workflow.Stop(c);
+            }
+            RunningGroups.Remove(group);
+            LogEnd("Stopped.");
+        }
+
+        public void DeleteAllResources()
+        {
+            var workflow = CreateWorkflow();
+            workflow.DeleteTestResources();
+
+            RunningGroups.Clear();
+        }
+
+        public void DownloadLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
+        {
+            var workflow = CreateWorkflow();
+            workflow.DownloadContainerLog(container, logHandler, tailLines);
+        }
+
+        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
+        {
+            if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory();
+
+            var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
+
+            if (codexSetup.MetricsMode == MetricsMode.Dashboard)
+            {
+                lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup);
+            }
+
+            return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
+        }
+
+        private StartupConfig CreateStartupConfig(GethStartResult gethStartResult, CodexSetup codexSetup)
+        {
+            var startupConfig = new StartupConfig();
+            startupConfig.NameOverride = codexSetup.NameOverride;
+            startupConfig.Add(codexSetup);
+            startupConfig.Add(gethStartResult);
+            return startupConfig;
+        }
+
+        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
+        {
+            var result = new List<RunningContainers>();
+            var recipe = new CodexContainerRecipe();
+            for (var i = 0; i < numberOfNodes; i++)
+            {
+                var workflow = CreateWorkflow();
+                var rc = workflow.Start(1, location, recipe, startupConfig);
+                CreateCrashWatcher(workflow, rc);
+                result.Add(rc);
+            }
+            return result.ToArray();
+        }
+
+        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
+        {
+            var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
+            RunningGroups.Add(group);
+
+            try
+            {
+                Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
+            }
+            catch
+            {
+                CodexNodesNotOnline(runningContainers);
+                throw;
+            }
+
+            return group;
+        }
+
+        private void CodexNodesNotOnline(RunningContainers[] runningContainers)
+        {
+            Log("Codex nodes failed to start");
+            foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
+        }
+
+        private StartupWorkflow CreateWorkflow()
+        {
+            return lifecycle.WorkflowCreator.CreateWorkflow();
+        }
+
+        private void LogSeparator()
+        {
+            Log("----------------------------------------------------------------------------");
+        }
+
+        private void CreateCrashWatcher(StartupWorkflow workflow, RunningContainers rc)
+        {
+            var c = rc.Containers.Single();
+            c.CrashWatcher = workflow.CreateCrashWatcher(c);
+        }
+
+        private void StopCrashWatcher(RunningContainers containers)
+        {
+            foreach (var c in containers.Containers)
+            {
+                c.CrashWatcher?.Stop();
+            }
+        }
+    }
+}

DistTestCore/Configuration.cs

@@ -0,0 +1,144 @@
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+using System.Net.NetworkInformation;
+using Utils;
+
+namespace DistTestCore
+{
+    public class Configuration
+    {
+        private readonly string? kubeConfigFile;
+        private readonly string logPath;
+        private readonly bool logDebug;
+        private readonly string dataFilesPath;
+        private readonly CodexLogLevel codexLogLevel;
+        private readonly string k8sNamespacePrefix;
+        private static RunnerLocation? runnerLocation = null;
+
+        public Configuration()
+        {
+            kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
+            logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
+            logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
+            dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
+            codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
+            k8sNamespacePrefix = "ct-";
+        }
+
+        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, string k8sNamespacePrefix)
+        {
+            this.kubeConfigFile = kubeConfigFile;
+            this.logPath = logPath;
+            this.logDebug = logDebug;
+            this.dataFilesPath = dataFilesPath;
+            this.codexLogLevel = codexLogLevel;
+            this.k8sNamespacePrefix = k8sNamespacePrefix;
+        }
+
+        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
+        {
+            return new KubernetesWorkflow.Configuration(
+                k8sNamespacePrefix: k8sNamespacePrefix,
+                kubeConfigFile: kubeConfigFile,
+                operationTimeout: timeSet.K8sOperationTimeout(),
+                retryDelay: timeSet.WaitForK8sServiceDelay()
+            );
+        }
+
+        public Logging.LogConfig GetLogConfig()
+        {
+            return new Logging.LogConfig(logPath, debugEnabled: logDebug);
+        }
+
+        public string GetFileManagerFolder()
+        {
+            return dataFilesPath;
+        }
+
+        public CodexLogLevel GetCodexLogLevel()
+        {
+            return codexLogLevel;
+        }
+
+        public Address GetAddress(RunningContainer container)
+        {
+            if (runnerLocation == null)
+            {
+                runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container);
+            }
+            
+            if (runnerLocation == RunnerLocation.InternalToCluster)
+            {
+                return container.ClusterInternalAddress;
+            }
+            return container.ClusterExternalAddress;
+        }
+
+        private static string GetEnvVarOrDefault(string varName, string defaultValue)
+        {
+            var v = Environment.GetEnvironmentVariable(varName);
+            if (v == null) return defaultValue;
+            return v;
+        }
+
+        private static string? GetNullableEnvVarOrDefault(string varName, string? defaultValue)
+        {
+            var v = Environment.GetEnvironmentVariable(varName);
+            if (v == null) return defaultValue;
+            return v;
+        }
+    }
+
+    public enum RunnerLocation
+    {
+        ExternalToCluster,
+        InternalToCluster,
+    }
+
+    public static class RunnerLocationUtils
+    {
+        private static bool alreadyDidThat = false;
+
+        public static RunnerLocation DetermineRunnerLocation(RunningContainer container)
+        {
+            // We want to be sure we don't ping more often than strictly necessary.
+            // If we have already determined the location during this application
+            // lifetime, don't do it again.
+            if (alreadyDidThat) throw new Exception("We already did that.");
+            alreadyDidThat = true;
+
+            if (PingHost(container.Pod.PodInfo.Ip))
+            {
+                return RunnerLocation.InternalToCluster;
+            }
+            if (PingHost(Format(container.ClusterExternalAddress)))
+            {
+                return RunnerLocation.ExternalToCluster;
+            }
+
+            throw new Exception("Unable to determine runner location.");
+        }
+
+        private static string Format(Address host)
+        {
+            return host.Host
+                .Replace("http://", "")
+                .Replace("https://", "");
+        }
+
+        private static bool PingHost(string host)
+        {
+            try
+            {
+                using var pinger = new Ping();
+                PingReply reply = pinger.Send(host);
+                return reply.Status == IPStatus.Success;
+            }
+            catch (PingException)
+            {
+            }
+
+            return false;
+        }
+    }
+}

DistTestCore/DefaultContainerRecipe.cs

@@ -0,0 +1,40 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace DistTestCore
+{
+    public abstract class DefaultContainerRecipe : ContainerRecipeFactory
+    {
+        public static string TestsType { get; set; } = "NotSet";
+        public static ApplicationIds? ApplicationIds { get; set; } = null;
+
+        protected abstract void InitializeRecipe(StartupConfig config);
+
+        protected override void Initialize(StartupConfig config)
+        {
+            Add("tests-type", TestsType);
+            Add("runid", NameUtils.GetRunId());
+            Add("testid", NameUtils.GetTestId());
+            Add("category", NameUtils.GetCategoryName());
+            Add("fixturename", NameUtils.GetRawFixtureName());
+            Add("testname", NameUtils.GetTestMethodName());
+
+            if (ApplicationIds != null)
+            {
+                Add("codexid", ApplicationIds.CodexId);
+                Add("gethid", ApplicationIds.GethId);
+                Add("prometheusid", ApplicationIds.PrometheusId);
+                Add("codexcontractsid", ApplicationIds.CodexContractsId);
+                Add("grafanaid", ApplicationIds.GrafanaId);
+            }
+            Add("app", AppName);
+
+            InitializeRecipe(config);
+        }
+
+        private void Add(string name, string value)
+        {
+            AddPodLabel(name, value);
+        }
+    }
+}

DistTestCore/DistTest.cs

@@ -1,49 +1,45 @@
-using Core;
+using DistTestCore.Codex;
+using DistTestCore.Helpers;
 using DistTestCore.Logs;
-using FileUtils;
+using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
+using KubernetesWorkflow;
 using Logging;
 using NUnit.Framework;
 using System.Reflection;
-using Utils;
-using Assert = NUnit.Framework.Assert;
 
 namespace DistTestCore
 {
     [Parallelizable(ParallelScope.All)]
     public abstract class DistTest
     {
-        private const string TestNamespacePrefix = "ct-";
+        private const string TestsType = "dist-tests";
         private readonly Configuration configuration = new Configuration();
         private readonly Assembly[] testAssemblies;
         private readonly FixtureLog fixtureLog;
         private readonly StatusLog statusLog;
         private readonly object lifecycleLock = new object();
-        private readonly EntryPoint globalEntryPoint;
         private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
-        private readonly string deployId;
-        
+
         public DistTest()
         {
             var assemblies = AppDomain.CurrentDomain.GetAssemblies();
             testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
-            
-            deployId = NameUtils.MakeDeployId();
 
             var logConfig = configuration.GetLogConfig();
             var startTime = DateTime.UtcNow;
-            fixtureLog = new FixtureLog(logConfig, startTime, deployId);
-            statusLog = new StatusLog(logConfig, startTime, "dist-tests", deployId);
-
-            globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder());
-
-            Initialize(fixtureLog);
+            fixtureLog = new FixtureLog(logConfig, startTime);
+            statusLog = new StatusLog(logConfig, startTime);
         }
 
         [OneTimeSetUp]
         public void GlobalSetup()
         {
-            fixtureLog.Log($"Distributed Tests are starting...");
-            globalEntryPoint.Announce();
+            fixtureLog.Log($"Codex Distributed Tests are starting...");
+            fixtureLog.Log($"Codex image: '{new CodexContainerRecipe().Image}'");
+            fixtureLog.Log($"CodexContracts image: '{new CodexContractsContainerRecipe().Image}'");
+            fixtureLog.Log($"Prometheus image: '{new PrometheusContainerRecipe().Image}'");
+            fixtureLog.Log($"Geth image: '{new GethContainerRecipe().Image}'");
 
             // Previous test run may have been interrupted.
             // Begin by cleaning everything up.
@@ -51,7 +47,8 @@ namespace DistTestCore
             {
                 Stopwatch.Measure(fixtureLog, "Global setup", () =>
                 {
-                    globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix);
+                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), string.Empty);
+                    wc.CreateWorkflow().DeleteAllResources();
                 });                
             }
             catch (Exception ex)
@@ -61,20 +58,9 @@ namespace DistTestCore
                 throw;
             }
 
-            fixtureLog.Log("Test framework revision: " + GitInfo.GetStatus());
             fixtureLog.Log("Global setup cleanup successful");
         }
 
-        [OneTimeTearDown]
-        public void GlobalTearDown()
-        {
-            globalEntryPoint.Decommission(
-                // There shouldn't be any of either, but clean everything up regardless.
-                deleteKubernetesResources: true,
-                deleteTrackedFiles: true
-            );
-        }
-
         [SetUp]
         public void SetUpDistTest()
         {
@@ -102,17 +88,9 @@ namespace DistTestCore
             }
         }
 
-        public CoreInterface Ci
+        public TestFile GenerateTestFile(ByteSize size, string label = "")
         {
-            get
-            {
-                return Get().CoreInterface;
-            }
-        }
-
-        public TrackedFile GenerateTestFile(ByteSize size, string label = "")
-        {
-            return Get().GenerateTestFile(size, label);
+            return Get().FileManager.GenerateTestFile(size, label);
         }
 
         /// <summary>
@@ -121,10 +99,60 @@ namespace DistTestCore
         /// </summary>
         public void ScopedTestFiles(Action action)
         {
-            Get().GetFileManager().ScopedFiles(action);
+            Get().FileManager.PushFileSet();
+            action();
+            Get().FileManager.PopFileSet();
         }
 
-        public ILog GetTestLog()
+        public IOnlineCodexNode SetupCodexBootstrapNode()
+        {
+            return SetupCodexBootstrapNode(s => { });
+        }
+
+        public virtual IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
+        {
+            return SetupCodexNode(s =>
+            {
+                setup(s);
+                s.WithName("Bootstrap");
+            });
+        }
+
+        public IOnlineCodexNode SetupCodexNode()
+        {
+            return SetupCodexNode(s => { });
+        }
+
+        public IOnlineCodexNode SetupCodexNode(Action<ICodexSetup> setup)
+        {
+            return SetupCodexNodes(1, setup)[0];
+        }
+
+        public ICodexNodeGroup SetupCodexNodes(int numberOfNodes)
+        {
+            return SetupCodexNodes(numberOfNodes, s => { });
+        }
+
+        public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        {
+            var codexSetup = CreateCodexSetup(numberOfNodes);
+
+            setup(codexSetup);
+
+            return BringOnline(codexSetup);
+        }
+
+        public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
+        {
+            return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
+        }
+
+        public IEnumerable<IOnlineCodexNode> GetAllOnlineCodexNodes()
+        {
+            return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
+        }
+
+        public BaseLog GetTestLog()
         {
             return Get().Log;
         }
@@ -141,33 +169,32 @@ namespace DistTestCore
             GetTestLog().Debug(msg);
         }
 
+        public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers()
+        {
+            return new PeerConnectionTestHelpers(GetTestLog());
+        }
+
+        public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers()
+        {
+            return new PeerDownloadTestHelpers(GetTestLog(), Get().FileManager);
+        }
+
+        public PeerFetchTestHelpers CreatePeerFetchTestHelpers()
+        {
+            return new PeerFetchTestHelpers(GetTestLog(), Get().FileManager);
+        }
+
         public void Measure(string name, Action action)
         {
             Stopwatch.Measure(Get().Log, name, action);
         }
 
-        protected TimeRange GetTestRunTimeRange()
+        protected CodexSetup CreateCodexSetup(int numberOfNodes)
         {
-            return new TimeRange(Get().TestStart, DateTime.UtcNow);
+            return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
         }
 
-        protected virtual void Initialize(FixtureLog fixtureLog)
-        {
-        }
-
-        protected virtual void LifecycleStart(TestLifecycle lifecycle)
-        {
-        }
-
-        protected virtual void LifecycleStop(TestLifecycle lifecycle)
-        {
-        }
-
-        protected virtual void CollectStatusLogData(TestLifecycle lifecycle, Dictionary<string, string> data)
-        {
-        }
-
-        protected TestLifecycle Get()
+        private TestLifecycle Get()
         {
             lock (lifecycleLock)
             {
@@ -183,10 +210,11 @@ namespace DistTestCore
             {
                 lock (lifecycleLock)
                 {
-                    var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString();
-                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace, deployId);
+                    var testNamespace = Guid.NewGuid().ToString();
+                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace);
                     lifecycles.Add(testName, lifecycle);
-                    LifecycleStart(lifecycle);
+                    DefaultContainerRecipe.TestsType = TestsType;
+                    DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
                 }
             });
         }
@@ -196,38 +224,17 @@ namespace DistTestCore
             var lifecycle = Get();
             var testResult = GetTestResult();
             var testDuration = lifecycle.GetTestDuration();
-            var data = lifecycle.GetPluginMetadata();
-            CollectStatusLogData(lifecycle, data);
             fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
-            statusLog.ConcludeTest(testResult, testDuration, data);
+            statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetApplicationIds());
             Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
             {
-                WriteEndTestLog(lifecycle.Log);
-
-                IncludeLogsOnTestFailure(lifecycle);
-                LifecycleStop(lifecycle);
+                lifecycle.Log.EndTest();
+                IncludeLogsAndMetricsOnTestFailure(lifecycle);
                 lifecycle.DeleteAllResources();
                 lifecycle = null!;
             });
         }
 
-        private void WriteEndTestLog(TestLog log)
-        {
-            var result = TestContext.CurrentContext.Result;
-
-            Log($"*** Finished: {GetCurrentTestName()} = {result.Outcome.Status}");
-            if (!string.IsNullOrEmpty(result.Message))
-            {
-                Log(result.Message);
-                Log($"{result.StackTrace}");
-            }
-
-            if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
-            {
-                log.MarkAsFailed();
-            }
-        }
-
         private ITimeSet GetTimeSet()
         {
             if (ShouldUseLongTimeouts()) return new LongTimeSet();
@@ -249,25 +256,57 @@ namespace DistTestCore
             return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
         }
 
-        private void IncludeLogsOnTestFailure(TestLifecycle lifecycle)
+        private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
         {
             var result = TestContext.CurrentContext.Result;
             if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
             {
                 fixtureLog.MarkAsFailed();
 
-                if (IsDownloadingLogsEnabled())
+                if (IsDownloadingLogsAndMetricsEnabled())
                 {
-                    lifecycle.Log.Log("Downloading all container logs because of test failure...");
-                    lifecycle.DownloadAllLogs();
+                    lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
+                    DownloadAllLogs(lifecycle);
+                    DownloadAllMetrics(lifecycle);
                 }
                 else
                 {
-                    lifecycle.Log.Log("Skipping download of all container logs due to [DontDownloadLogsOnFailure] attribute.");
+                    lifecycle.Log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
                 }
             }
         }
 
+        private void DownloadAllLogs(TestLifecycle lifecycle)
+        {
+            OnEachCodexNode(lifecycle, node =>
+            {
+                lifecycle.DownloadLog(node.CodexAccess.Container);
+            });
+        }
+
+        private void DownloadAllMetrics(TestLifecycle lifecycle)
+        {
+            var metricsDownloader = new MetricsDownloader(lifecycle.Log);
+
+            OnEachCodexNode(lifecycle, node =>
+            {
+                var m = node.Metrics as MetricsAccess;
+                if (m != null)
+                {
+                    metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m);
+                }
+            });
+        }
+
+        private void OnEachCodexNode(TestLifecycle lifecycle, Action<OnlineCodexNode> action)
+        {
+            var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
+            foreach (var node in allNodes)
+            {
+                action(node);
+            }
+        }
+
         private string GetCurrentTestName()
         {
             return $"[{TestContext.CurrentContext.Test.Name}]";
@@ -278,10 +317,10 @@ namespace DistTestCore
             return TestContext.CurrentContext.Result.Outcome.Status.ToString();
         }
 
-        private bool IsDownloadingLogsEnabled()
+        private bool IsDownloadingLogsAndMetricsEnabled()
         {
             var testProperties = TestContext.CurrentContext.Test.Properties;
-            return !testProperties.ContainsKey(DontDownloadLogsOnFailureAttribute.DontDownloadKey);
+            return !testProperties.ContainsKey(DontDownloadLogsAndMetricsOnFailureAttribute.DontDownloadKey);
         }
     }
 

DistTestCore/DistTestCore.csproj

@@ -0,0 +1,34 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+	  <RootNamespace>DistTestCore</RootNamespace>
+	  <ImplicitUsings>enable</ImplicitUsings>
+	  <Nullable>enable</Nullable>
+    <IsArm64 Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)' == 'Arm64'">true</IsArm64>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(IsArm64)'=='true'">
+    <DefineConstants>Arm64</DefineConstants>
+  </PropertyGroup>
+  <ItemGroup>
+    <None Remove="Metrics\dashboard.json" />
+  </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="Metrics\dashboard.json">
+      <CopyToOutputDirectory>Never</CopyToOutputDirectory>
+    </EmbeddedResource>
+  </ItemGroup>
+
+  <ItemGroup>
+	  <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+	  <PackageReference Include="nunit" Version="3.13.3" />
+	  <PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
+	  <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+    <ProjectReference Include="..\Nethereum\NethereumWorkflow.csproj" />
+  </ItemGroup>
+</Project>

DistTestCore/FileManager.cs

@@ -0,0 +1,218 @@
+using Logging;
+using NUnit.Framework;
+using Utils;
+
+namespace DistTestCore
+{
+    public interface IFileManager
+    {
+        TestFile CreateEmptyTestFile(string label = "");
+        TestFile GenerateTestFile(ByteSize size, string label = "");
+        void DeleteAllTestFiles();
+        void PushFileSet();
+        void PopFileSet();
+    }
+
+    public class FileManager : IFileManager
+    {
+        public const int ChunkSize = 1024 * 1024 * 100;
+        private static NumberSource folderNumberSource = new NumberSource(0);
+        private readonly Random random = new Random();
+        private readonly BaseLog log;
+        private readonly string folder;
+        private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();
+
+        public FileManager(BaseLog log, Configuration configuration)
+        {
+            folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5"));
+
+            EnsureDirectory();
+            this.log = log;
+        }
+
+        public TestFile CreateEmptyTestFile(string label = "")
+        {
+            var path = Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin");
+            var result = new TestFile(log, path, label);
+            File.Create(result.Filename).Close();
+            if (fileSetStack.Any()) fileSetStack.Last().Add(result);
+            return result;
+        }
+
+        public TestFile GenerateTestFile(ByteSize size, string label)
+        {
+            var sw = Stopwatch.Begin(log);
+            var result = GenerateFile(size, label);
+            sw.End($"Generated file '{result.Describe()}'.");
+            return result;
+        }
+
+        public void DeleteAllTestFiles()
+        {
+            DeleteDirectory();
+        }
+
+        public void PushFileSet()
+        {
+            fileSetStack.Add(new List<TestFile>());
+        }
+
+        public void PopFileSet()
+        {
+            if (!fileSetStack.Any()) return;
+            var pop = fileSetStack.Last();
+            fileSetStack.Remove(pop);
+
+            foreach (var file in pop)
+            {
+                try
+                {
+                    File.Delete(file.Filename);
+                }
+                catch { }
+            }
+        }
+
+        private TestFile GenerateFile(ByteSize size, string label)
+        {
+            var result = CreateEmptyTestFile(label);
+            CheckSpaceAvailable(result, size);
+
+            GenerateFileBytes(result, size);
+            return result;
+        }
+
+        private void CheckSpaceAvailable(TestFile testFile, ByteSize size)
+        {
+            var file = new FileInfo(testFile.Filename);
+            var drive = new DriveInfo(file.Directory!.Root.FullName);
+
+            var spaceAvailable = drive.TotalFreeSpace;
+
+            if (spaceAvailable < size.SizeInBytes)
+            {
+                var msg = $"Inconclusive: Not enough disk space to perform test. " +
+                    $"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
+                    $"{Formatter.FormatByteSize(spaceAvailable)} available.";
+
+                log.Log(msg);
+                Assert.Inconclusive(msg);
+            }
+        }
+
+        private void GenerateFileBytes(TestFile result, ByteSize size)
+        {
+            long bytesLeft = size.SizeInBytes;
+            int chunkSize = ChunkSize;
+            while (bytesLeft > 0)
+            {
+                try
+                {
+                    var length = Math.Min(bytesLeft, chunkSize);
+                    AppendRandomBytesToFile(result, length);
+                    bytesLeft -= length;
+                }
+                catch
+                {
+                    chunkSize = chunkSize / 2;
+                    if (chunkSize < 1024) throw;
+                }
+            }
+        }
+
+        private void AppendRandomBytesToFile(TestFile result, long length)
+        {
+            var bytes = new byte[length];
+            random.NextBytes(bytes);
+            using var stream = new FileStream(result.Filename, FileMode.Append);
+            stream.Write(bytes, 0, bytes.Length);
+        }
+
+        private void EnsureDirectory()
+        {
+            if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
+        }
+
+        private void DeleteDirectory()
+        {
+            Directory.Delete(folder, true);
+        }
+    }
+
+    public class TestFile
+    {
+        private readonly BaseLog log;
+
+        public TestFile(BaseLog log, string filename, string label)
+        {
+            this.log = log;
+            Filename = filename;
+            Label = label;
+        }
+
+        public string Filename { get; }
+        public string Label { get; }
+
+        public void AssertIsEqual(TestFile? actual)
+        {
+            var sw = Stopwatch.Begin(log);
+            try
+            {
+                AssertEqual(actual);
+            }
+            finally
+            {
+                sw.End($"{nameof(TestFile)}.{nameof(AssertIsEqual)}");
+            }
+        }
+
+        public string Describe()
+        {
+            var sizePostfix = $" ({Formatter.FormatByteSize(GetFileSize())})";
+            if (!string.IsNullOrEmpty(Label)) return Label + sizePostfix;
+            return $"'{Filename}'{sizePostfix}";
+        }
+
+        private void AssertEqual(TestFile? actual)
+        {
+            if (actual == null) Assert.Fail("TestFile is null.");
+            if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
+
+            Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length.");
+
+            using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
+            using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
+
+            var bytesExpected = new byte[FileManager.ChunkSize];
+            var bytesActual = new byte[FileManager.ChunkSize];
+
+            var readExpected = 0;
+            var readActual = 0;
+
+            while (true)
+            {
+                readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
+                readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
+
+                if (readExpected == 0 && readActual == 0)
+                {
+                    log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
+                    return;
+                }
+
+                Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length.");
+
+                for (var i = 0; i < readActual; i++)
+                {
+                    if (bytesExpected[i] != bytesActual[i]) Assert.Fail("File contents not equal.");
+                }
+            }
+        }
+
+        private long GetFileSize()
+        {
+            var info = new FileInfo(Filename);
+            return info.Length;
+        }
+    }
+}

DistTestCore/GethStarter.cs

@@ -0,0 +1,88 @@
+using DistTestCore.Marketplace;
+
+namespace DistTestCore
+{
+    public class GethStarter : BaseStarter
+    {
+        private readonly MarketplaceNetworkCache marketplaceNetworkCache;
+        private readonly GethCompanionNodeStarter companionNodeStarter;
+
+        public GethStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
+        {
+            marketplaceNetworkCache = new MarketplaceNetworkCache(
+                new GethBootstrapNodeStarter(lifecycle),
+                new CodexContractsStarter(lifecycle));
+            companionNodeStarter = new GethCompanionNodeStarter(lifecycle);
+        }
+
+        public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup)
+        {
+            if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult();
+
+            var marketplaceNetwork = marketplaceNetworkCache.Get();
+            var companionNode = StartCompanionNode(codexSetup, marketplaceNetwork);
+
+            LogStart("Setting up initial balance...");
+            TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNode);
+            LogEnd($"Initial balance of {codexSetup.MarketplaceConfig.InitialTestTokens} set for {codexSetup.NumberOfNodes} nodes.");
+
+            return CreateGethStartResult(marketplaceNetwork, companionNode);
+        }
+
+        private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
+        {
+            if (marketplaceConfig.InitialTestTokens.Amount == 0) return;
+
+            var interaction = marketplaceNetwork.StartInteraction(lifecycle);
+            var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
+
+            var accounts = companionNode.Accounts.Select(a => a.Account).ToArray();
+            interaction.MintTestTokens(accounts, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
+        }
+
+        private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
+        {
+            return new GethStartResult(CreateMarketplaceAccessFactory(marketplaceNetwork), marketplaceNetwork, companionNode);
+        }
+
+        private GethStartResult CreateMarketplaceUnavailableResult()
+        {
+            return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, null!);
+        }
+
+        private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork)
+        {
+            return new GethMarketplaceAccessFactory(lifecycle, marketplaceNetwork);
+        }
+
+        private GethCompanionNodeInfo StartCompanionNode(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)
+        {
+            return companionNodeStarter.StartCompanionNodeFor(codexSetup, marketplaceNetwork);
+        }
+    }
+
+    public class MarketplaceNetworkCache
+    {
+        private readonly GethBootstrapNodeStarter bootstrapNodeStarter;
+        private readonly CodexContractsStarter codexContractsStarter;
+        private MarketplaceNetwork? network;
+
+        public MarketplaceNetworkCache(GethBootstrapNodeStarter bootstrapNodeStarter, CodexContractsStarter codexContractsStarter)
+        {
+            this.bootstrapNodeStarter = bootstrapNodeStarter;
+            this.codexContractsStarter = codexContractsStarter;
+        }
+
+        public MarketplaceNetwork Get()
+        {
+            if (network == null)
+            {
+                var bootstrapInfo = bootstrapNodeStarter.StartGethBootstrapNode();
+                var marketplaceInfo = codexContractsStarter.Start(bootstrapInfo);
+                network = new MarketplaceNetwork(bootstrapInfo, marketplaceInfo );
+            }
+            return network;
+        }
+    }
+}

DistTestCore/GrafanaStarter.cs

@@ -0,0 +1,192 @@
+using DistTestCore.Metrics;
+using IdentityModel.Client;
+using KubernetesWorkflow;
+using Newtonsoft.Json;
+using System.Reflection;
+
+namespace DistTestCore
+{
+    public class GrafanaStarter : BaseStarter
+    {
+        private const string StorageQuotaThresholdReplaceToken = "\"<CODEX_STORAGEQUOTA>\"";
+        private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"<CODEX_BYTESUSED_SOFTMAX>\"";
+
+        public GrafanaStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
+        {
+        }
+
+        public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup)
+        {
+            LogStart($"Starting dashboard server");
+
+            var grafanaContainer = StartGrafanaContainer();
+            var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer);
+
+            var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth);
+
+            Log("Connecting datasource...");
+            AddDataSource(http, prometheusContainer);
+
+            Log("Uploading dashboard configurations...");
+            var jsons = ReadEachDashboardJsonFile(codexSetup);
+            var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
+
+            LogEnd("Dashboard server started.");
+
+            return new GrafanaStartInfo(dashboardUrls, grafanaContainer);
+        }
+
+        private RunningContainer StartGrafanaContainer()
+        {
+            var startupConfig = new StartupConfig();
+
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig);
+            if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created.");
+
+            return grafanaContainers.Containers.First();
+        }
+
+        private void AddBasicAuth(HttpClient client)
+        {
+            client.SetBasicAuthentication(
+                GrafanaContainerRecipe.DefaultAdminUser,
+                GrafanaContainerRecipe.DefaultAdminPassword);
+        }
+
+        private static void AddDataSource(Http http, RunningContainer prometheusContainer)
+        {
+            var prometheusAddress = prometheusContainer.ClusterExternalAddress;
+            var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port;
+            var response = http.HttpPostJson<GrafanaDataSourceRequest, GrafanaDataSourceResponse>("datasources", new GrafanaDataSourceRequest
+            {
+                uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb",
+                name = "CodexPrometheus",
+                type = "prometheus",
+                url = prometheusUrl,
+                access = "proxy",
+                basicAuth = false,
+                jsonData = new GrafanaDataSourceJsonData
+                {
+                    httpMethod = "POST"
+                }
+            });
+
+            if (response.message != "Datasource added")
+            {
+                throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message);
+            }
+        }
+
+        public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson)
+        {
+            var request = GetDashboardCreateRequest(dashboardJson);
+            var response = http.HttpPostString("dashboards/db", request);
+            var jsonResponse = JsonConvert.DeserializeObject<GrafanaPostDashboardResponse>(response);
+            if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard.");
+
+            var grafanaAddress = grafanaContainer.ClusterExternalAddress;
+            return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
+        }
+
+        private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup)
+        {
+            var assembly = Assembly.GetExecutingAssembly();
+            var resourceNames = new[]
+            {
+                "DistTestCore.Metrics.dashboard.json"
+            };
+
+            return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray();
+        }
+
+        private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup)
+        {
+            using var stream = assembly.GetManifestResourceStream(resourceName);
+            if (stream == null) throw new Exception("Unable to find resource " + resourceName);
+            using var reader = new StreamReader(stream);
+            return ApplyReplacements(reader.ReadToEnd(), codexSetup);
+        }
+
+        private static string ApplyReplacements(string input, CodexSetup codexSetup)
+        {
+            var quotaString = GetQuotaString(codexSetup);
+            var softMaxString = GetSoftMaxString(codexSetup);
+
+            return input
+                .Replace(StorageQuotaThresholdReplaceToken, quotaString)
+                .Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString);
+        }
+
+        private static string GetQuotaString(CodexSetup codexSetup)
+        {
+            return GetCodexStorageQuotaInBytes(codexSetup).ToString();
+        }
+
+        private static string GetSoftMaxString(CodexSetup codexSetup)
+        {
+            var quota = GetCodexStorageQuotaInBytes(codexSetup);
+            var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing.
+            return softMax.ToString();
+        }
+
+        private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup)
+        {
+            if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes;
+
+            // Codex default: 8GB
+            return 8.GB().SizeInBytes;
+        }
+
+        private static string GetDashboardCreateRequest(string dashboardJson)
+        {
+            return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}";
+        }
+    }
+
+    public class GrafanaStartInfo
+    {
+        public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container)
+        {
+            DashboardUrls = dashboardUrls;
+            Container = container;
+        }
+
+        public string[] DashboardUrls { get; }
+        public RunningContainer Container { get; }
+    }
+
+    public class GrafanaDataSourceRequest
+    {
+        public string uid { get; set; } = string.Empty;
+        public string name { get; set; } = string.Empty;
+        public string type { get; set; } = string.Empty;
+        public string url { get; set; } = string.Empty;
+        public string access { get; set; } = string.Empty;
+        public bool basicAuth { get; set; }
+        public GrafanaDataSourceJsonData jsonData { get; set; } = new();
+    }
+
+    public class GrafanaDataSourceResponse
+    {
+        public int id { get; set; }
+        public string message { get; set; } = string.Empty;
+        public string name { get; set; } = string.Empty;
+    }
+
+    public class GrafanaDataSourceJsonData
+    {
+        public string httpMethod { get; set; } = string.Empty;
+    }
+
+    public class GrafanaPostDashboardResponse
+    {
+        public int id { get; set; }
+        public string slug { get; set; } = string.Empty;
+        public string status { get; set; } = string.Empty;
+        public string uid { get; set; } = string.Empty;
+        public string url { get; set; } = string.Empty;
+        public int version { get; set; }
+    }
+}

DistTestCore/Helpers/AssertHelpers.cs

@@ -10,11 +10,8 @@ namespace DistTestCore.Helpers
         {
             try
             {
-
-                Time.WaitUntil(() => {
-                    var c = constraint.Resolve();
-                    return c.ApplyTo(actual()).IsSuccess;
-                });
+                var c = constraint.Resolve();
+                Time.WaitUntil(() => c.ApplyTo(actual()).IsSuccess);
             }
             catch (TimeoutException)
             {

DistTestCore/Helpers/FullConnectivityHelper.cs

@@ -1,8 +1,9 @@
-using CodexPlugin;
+using DistTestCore.Codex;
 using Logging;
 using NUnit.Framework;
+using Utils;
 
-namespace CodexTests.Helpers
+namespace DistTestCore.Helpers
 {
     public interface IFullConnectivityImplementation
     {
@@ -14,29 +15,30 @@ namespace CodexTests.Helpers
     public class FullConnectivityHelper
     {
         private static string Nl = Environment.NewLine;
-        private readonly ILog log;
+        private readonly BaseLog log;
         private readonly IFullConnectivityImplementation implementation;
 
-        public FullConnectivityHelper(ILog log, IFullConnectivityImplementation implementation)
+        public FullConnectivityHelper(BaseLog log, IFullConnectivityImplementation implementation)
         {
             this.log = log;
             this.implementation = implementation;
         }
 
-        public void AssertFullyConnected(IEnumerable<ICodexNode> nodes)
+        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
         {
             AssertFullyConnected(nodes.ToArray());
         }
 
-        private void AssertFullyConnected(ICodexNode[] nodes)
+        private void AssertFullyConnected(CodexAccess[] nodes)
         {
             Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
             var entries = CreateEntries(nodes);
             var pairs = CreatePairs(entries);
 
-            // Each pair gets two chances.
-            CheckAndRemoveSuccessful(pairs);
-            CheckAndRemoveSuccessful(pairs);
+            RetryWhilePairs(pairs, () =>
+            {
+                CheckAndRemoveSuccessful(pairs);
+            });
 
             if (pairs.Any())
             {
@@ -52,22 +54,38 @@ namespace CodexTests.Helpers
             }
         }
 
+        private static void RetryWhilePairs(List<Pair> pairs, Action action)
+        {
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2);
+            while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
+            {
+                action();
+
+                Time.Sleep(TimeSpan.FromSeconds(2));
+            }
+        }
+
         private void CheckAndRemoveSuccessful(List<Pair> pairs)
         {
-            var results = new List<string>();
-            foreach (var pair in pairs.ToArray())
+            // For large sets, don't try and do all of them at once.
+            var selectedPair = pairs.Take(20).ToArray();
+            var pairDetails = new List<string>();
+
+            foreach (var pair in selectedPair)
             {
                 pair.Check();
+
                 if (pair.Success)
                 {
-                    results.AddRange(pair.GetResultMessages());
+                    pairDetails.AddRange(pair.GetResultMessages());
                     pairs.Remove(pair);
                 }
             }
-            Log($"Connections successful:{Nl}{string.Join(Nl, results)}");
+
+            Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
         }
 
-        private Entry[] CreateEntries(ICodexNode[] nodes)
+        private Entry[] CreateEntries(CodexAccess[] nodes)
         {
             var entries = nodes.Select(n => new Entry(n)).ToArray();
 
@@ -107,13 +125,13 @@ namespace CodexTests.Helpers
 
         public class Entry
         {
-            public Entry(ICodexNode node)
+            public Entry(CodexAccess node)
             {
                 Node = node;
                 Response = node.GetDebugInfo();
             }
 
-            public ICodexNode Node { get; }
+            public CodexAccess Node { get; }
             public CodexDebugResponse Response { get; }
 
             public override string ToString()

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -1,19 +1,24 @@
-using CodexPlugin;
+using DistTestCore.Codex;
 using Logging;
-using static CodexTests.Helpers.FullConnectivityHelper;
+using static DistTestCore.Helpers.FullConnectivityHelper;
 
-namespace CodexTests.Helpers
+namespace DistTestCore.Helpers
 {
     public class PeerConnectionTestHelpers : IFullConnectivityImplementation
     {
         private readonly FullConnectivityHelper helper;
 
-        public PeerConnectionTestHelpers(ILog log)
+        public PeerConnectionTestHelpers(BaseLog log)
         {
             helper = new FullConnectivityHelper(log, this);
         }
 
-        public void AssertFullyConnected(IEnumerable<ICodexNode> nodes)
+        public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullyConnected(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess));
+        }
+
+        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
         {
             helper.AssertFullyConnected(nodes);
         }
@@ -58,10 +63,8 @@ namespace CodexTests.Helpers
             var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
             if (peer == null) return $"peerId: {node.peerId} is not known.";
 
-            var container = peer.Node.Container;
-            var podInfo = peer.Node.GetPodInfo();
-            var ip = podInfo.Ip;
-            var discPort = container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag)!;
+            var ip = peer.Node.Container.Pod.PodInfo.Ip;
+            var discPort = peer.Node.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
             return $"{ip}:{discPort.Number}";
         }
     }

DistTestCore/Helpers/PeerDownloadTestHelpers.cs

@@ -1,25 +1,30 @@
-using CodexPlugin;
-using FileUtils;
+using DistTestCore.Codex;
 using Logging;
-using Utils;
-using static CodexTests.Helpers.FullConnectivityHelper;
+using static DistTestCore.Helpers.FullConnectivityHelper;
 
-namespace CodexTests.Helpers
+namespace DistTestCore.Helpers
 {
     public class PeerDownloadTestHelpers : IFullConnectivityImplementation
     {
         private readonly FullConnectivityHelper helper;
-        private readonly IFileManager fileManager;
+        private readonly BaseLog log;
+        private readonly FileManager fileManager;
         private ByteSize testFileSize;
 
-        public PeerDownloadTestHelpers(ILog log, IFileManager fileManager)
+        public PeerDownloadTestHelpers(BaseLog log, FileManager fileManager)
         {
             helper = new FullConnectivityHelper(log, this);
             testFileSize = 1.MB();
+            this.log = log;
             this.fileManager = fileManager;
         }
 
-        public void AssertFullDownloadInterconnectivity(IEnumerable<ICodexNode> nodes, ByteSize testFileSize)
+        public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
+        {
+            AssertFullDownloadInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess), testFileSize);
+        }
+
+        public void AssertFullDownloadInterconnectivity(IEnumerable<CodexAccess> nodes, ByteSize testFileSize)
         {
             this.testFileSize = testFileSize;
             helper.AssertFullyConnected(nodes);
@@ -37,17 +42,15 @@ namespace CodexTests.Helpers
 
         public PeerConnectionState Check(Entry from, Entry to)
         {
-            return fileManager.ScopedFiles(() => CheckConnectivity(from, to));
-        }
-
-        private PeerConnectionState CheckConnectivity(Entry from, Entry to)
-        {
+            fileManager.PushFileSet();
             var expectedFile = GenerateTestFile(from.Node, to.Node);
-            var contentId = from.Node.UploadFile(expectedFile);
+
+            using var uploadStream = File.OpenRead(expectedFile.Filename);
+            var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(uploadStream));
 
             try
             {
-                var downloadedFile = DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded");
+                var downloadedFile = Stopwatch.Measure(log, "Download", () => DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded"));
                 expectedFile.AssertIsEqual(downloadedFile);
                 return PeerConnectionState.Connection;
             }
@@ -57,20 +60,29 @@ namespace CodexTests.Helpers
                 // We consider that as no-connection for the purpose of this test.
                 return PeerConnectionState.NoConnection;
             }
+            finally
+            {
+                fileManager.PopFileSet();
+            }
+
             // Should an exception occur during upload, then this try is inconclusive and we try again next loop.
         }
 
-        private TrackedFile? DownloadFile(ICodexNode node, ContentId contentId, string label)
+        private TestFile DownloadFile(CodexAccess node, string contentId, string label)
         {
-            return node.DownloadContent(contentId, label);
+            var downloadedFile = fileManager.CreateEmptyTestFile(label);
+            using var downloadStream = File.OpenWrite(downloadedFile.Filename);
+            using var stream = node.DownloadFile(contentId);
+            stream.CopyTo(downloadStream);
+            return downloadedFile;
         }
 
-        private TrackedFile GenerateTestFile(ICodexNode uploader, ICodexNode downloader)
+        private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader)
         {
             var up = uploader.GetName().Replace("<", "").Replace(">", "");
             var down = downloader.GetName().Replace("<", "").Replace(">", "");
             var label = $"~from:{up}-to:{down}~";
-            return fileManager.GenerateFile(testFileSize, label);
+            return fileManager.GenerateTestFile(testFileSize, label);
         }
     }
 }

DistTestCore/Helpers/PeerFetchTestHelpers.cs

@@ -0,0 +1,96 @@
+using DistTestCore.Codex;
+using Logging;
+using NUnit.Framework;
+using static DistTestCore.Helpers.FullConnectivityHelper;
+
+namespace DistTestCore.Helpers
+{
+    public class PeerFetchTestHelpers : IFullConnectivityImplementation
+    {
+        private readonly FullConnectivityHelper helper;
+        private readonly BaseLog log;
+        private readonly IFileManager fileManager;
+        private readonly ByteSize testFileSize;
+        private readonly int expectedNumberOfBlocks;
+
+        public PeerFetchTestHelpers(BaseLog log, IFileManager fileManager)
+        {
+            helper = new FullConnectivityHelper(log, this);
+            testFileSize = 10.MB();
+            expectedNumberOfBlocks = 161;
+            this.log = log;
+            this.fileManager = fileManager;
+        }
+
+        public void AssertFullFetchInterconnectivity(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullFetchInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess));
+        }
+
+        public void AssertFullFetchInterconnectivity(IEnumerable<CodexAccess> nodes)
+        {
+            helper.AssertFullyConnected(nodes);
+        }
+
+        public string Description()
+        {
+            return "Fetch connectivity";
+        }
+
+        public string ValidateEntry(Entry entry, Entry[] allEntries)
+        {
+            return string.Empty;
+        }
+
+        public PeerConnectionState Check(Entry from, Entry to)
+        {
+            fileManager.PushFileSet();
+            var expectedFile = GenerateTestFile(from.Node, to.Node);
+
+            using var uploadStream = File.OpenRead(expectedFile.Filename);
+            var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(uploadStream));
+            var originalFetch = from.Node.DebugFetch(contentId);
+            Assert.That(Convert.ToInt32(originalFetch.numberOfBlocks), Is.EqualTo(expectedNumberOfBlocks));
+            Assert.That(originalFetch.originalBytes.Replace("'NByte", ""), Is.EqualTo("10485760"));
+
+            try
+            {
+                var fetchResponse = Stopwatch.Measure(log, "Fetch", () => to.Node.DebugFetch(contentId));
+                AssertEqual(originalFetch, fetchResponse);
+                return PeerConnectionState.Connection;
+            }
+            catch
+            {
+                return PeerConnectionState.NoConnection;
+            }
+            finally
+            {
+                fileManager.PopFileSet();
+            }
+        }
+
+        private void AssertEqual(CodexDebugFetchResponse expected, CodexDebugFetchResponse actual)
+        {
+            Assert.That(actual.originalBytes, Is.EqualTo(expected.originalBytes));
+            Assert.That(actual.blockSize, Is.EqualTo(expected.blockSize));
+            Assert.That(actual.numberOfBlocks, Is.EqualTo(expected.numberOfBlocks));
+            Assert.That(actual.version, Is.EqualTo(expected.version));
+            Assert.That(actual.hcodec, Is.EqualTo(expected.hcodec));
+            Assert.That(actual.codec, Is.EqualTo(expected.codec));
+            Assert.That(actual.@protected, Is.EqualTo(expected.@protected));
+            Assert.That(actual.blocks.Length, Is.EqualTo(expected.blocks.Length));
+            for (var i = 0; i < expected.blocks.Length; i++)
+            {
+                Assert.That(actual.blocks[i].cid, Is.EqualTo(expected.blocks[i].cid));
+            }
+        }
+
+        private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader)
+        {
+            var up = uploader.GetName().Replace("<", "").Replace(">", "");
+            var down = downloader.GetName().Replace("<", "").Replace(">", "");
+            var label = $"~from:{up}-to:{down}~";
+            return fileManager.GenerateTestFile(testFileSize, label);
+        }
+    }
+}

DistTestCore/Http.cs

@@ -5,36 +5,23 @@ using System.Net.Http.Headers;
 using System.Net.Http.Json;
 using Utils;
 
-namespace Core
+namespace DistTestCore
 {
-    public interface IHttp
+    public class Http
     {
-        string HttpGetString(string route);
-        T HttpGetJson<T>(string route);
-        TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
-        string HttpPostJson<TRequest>(string route, TRequest body);
-        TResponse HttpPostString<TResponse>(string route, string body);
-        string HttpPostStream(string route, Stream stream);
-        Stream HttpGetStream(string route);
-        T Deserialize<T>(string json);
-    }
-
-    internal class Http : IHttp
-    {
-        private static readonly object httpLock = new object();
-        private readonly ILog log;
+        private readonly BaseLog log;
         private readonly ITimeSet timeSet;
         private readonly Address address;
         private readonly string baseUrl;
         private readonly Action<HttpClient> onClientCreated;
         private readonly string? logAlias;
 
-        internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
+        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
             : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
         {
         }
 
-        internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
+        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
         {
             this.log = log;
             this.timeSet = timeSet;
@@ -48,60 +35,61 @@ namespace Core
 
         public string HttpGetString(string route)
         {
-            return LockRetry(() =>
+            return Retry(() =>
             {
-                return GetString(route);
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                Log(url, "");
+                var result = Time.Wait(client.GetAsync(url));
+                var str = Time.Wait(result.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str; ;
             }, $"HTTP-GET:{route}");
         }
 
         public T HttpGetJson<T>(string route)
         {
-            return LockRetry(() =>
-            {
-                var json = GetString(route);
-                return Deserialize<T>(json);
-            }, $"HTTP-GET:{route}");
+            var json = HttpGetString(route);
+            return TryJsonDeserialize<T>(json);
         }
 
         public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
         {
-            return LockRetry(() =>
+            var response = PostJson(route, body);
+            var json = Time.Wait(response.Content.ReadAsStringAsync());
+            if (!response.IsSuccessStatusCode)
             {
-                var response = PostJson(route, body);
-                var json = Time.Wait(response.Content.ReadAsStringAsync());
-                if (!response.IsSuccessStatusCode)
-                {
-                    throw new HttpRequestException(json);
-                }
-                Log(GetUrl() + route, json);
-                return Deserialize<TResponse>(json);
-            }, $"HTTP-POST-JSON: {route}");
+                throw new HttpRequestException(json);
+            }
+            Log(GetUrl() + route, json);
+            return TryJsonDeserialize<TResponse>(json);
         }
 
         public string HttpPostJson<TRequest>(string route, TRequest body)
         {
-            return LockRetry(() =>
-            {
-                var response = PostJson(route, body);
-                return Time.Wait(response.Content.ReadAsStringAsync());
-            }, $"HTTP-POST-JSON: {route}");
+            var response = PostJson<TRequest>(route, body);
+            return Time.Wait(response.Content.ReadAsStringAsync());
         }
 
-        public TResponse HttpPostString<TResponse>(string route, string body)
+        public string HttpPostString(string route, string body)
         {
-            return LockRetry(() =>
+            return Retry(() =>
             {
-                var response = PostJsonString(route, body);
-                if (response == null) throw new Exception("Received no response.");
-                var result = Deserialize<TResponse>(response);
-                if (result == null) throw new Exception("Failed to deserialize response");
-                return result;
-            }, $"HTTO-POST-JSON: {route}");
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                Log(url, body);
+                var content = new StringContent(body);
+                content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+                var result = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(result.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STRING: {route}");
         }
 
         public string HttpPostStream(string route, Stream stream)
         {
-            return LockRetry(() =>
+            return Retry(() =>
             {
                 using var client = GetClient();
                 var url = GetUrl() + route;
@@ -117,7 +105,7 @@ namespace Core
 
         public Stream HttpGetStream(string route)
         {
-            return LockRetry(() =>
+            return Retry(() =>
             {
                 var client = GetClient();
                 var url = GetUrl() + route;
@@ -126,7 +114,7 @@ namespace Core
             }, $"HTTP-GET-STREAM: {route}");
         }
 
-        public T Deserialize<T>(string json)
+        public T TryJsonDeserialize<T>(string json)
         {
             var errors = new List<string>();
             var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
@@ -144,7 +132,7 @@ namespace Core
                     }
                 }
             });
-            if (errors.Count > 0)
+            if (errors.Count() > 0)
             {
                 throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
             }
@@ -155,37 +143,16 @@ namespace Core
             return deserialized;
         }
 
-        private string GetString(string route)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            Log(url, "");
-            var result = Time.Wait(client.GetAsync(url));
-            var str = Time.Wait(result.Content.ReadAsStringAsync());
-            Log(url, str);
-            return str;
-        }
-
         private HttpResponseMessage PostJson<TRequest>(string route, TRequest body)
         {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            using var content = JsonContent.Create(body);
-            Log(url, JsonConvert.SerializeObject(body));
-            return Time.Wait(client.PostAsync(url, content));
-        }
-
-        private string PostJsonString(string route, string body)
-        {
-            using var client = GetClient();
-            var url = GetUrl() + route;
-            Log(url, body);
-            var content = new StringContent(body);
-            content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
-            var result = Time.Wait(client.PostAsync(url, content));
-            var str = Time.Wait(result.Content.ReadAsStringAsync());
-            Log(url, str);
-            return str;
+            return Retry(() =>
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                using var content = JsonContent.Create(body);
+                Log(url, JsonConvert.SerializeObject(body));
+                return Time.Wait(client.PostAsync(url, content));
+            }, $"HTTP-POST-JSON: {route}");
         }
 
         private string GetUrl()
@@ -205,12 +172,9 @@ namespace Core
             }
         }
 
-        private T LockRetry<T>(Func<T> operation, string description)
+        private T Retry<T>(Func<T> operation, string description)
         {
-            lock (httpLock)
-            {
-                return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description);
-            }
+            return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
         }
 
         private HttpClient GetClient()

DistTestCore/Logs/DontDownloadLogsAndMetricsOnFailureAttribute.cs

@@ -0,0 +1,15 @@
+using NUnit.Framework;
+
+namespace DistTestCore.Logs
+{
+    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+    public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute
+    {
+        public const string DontDownloadKey = "DontDownloadLogsAndMetrics";
+
+        public DontDownloadLogsAndMetricsOnFailureAttribute()
+            : base(DontDownloadKey)
+        {
+        }
+    }
+}

DistTestCore/Logs/DownloadedLog.cs

@@ -1,40 +1,39 @@
 using Logging;
+using NUnit.Framework;
 
-namespace Core
+namespace DistTestCore.Logs
 {
     public interface IDownloadedLog
     {
-        string[] GetLinesContaining(string expectedString);
+        void AssertLogContains(string expectedString);
         string[] FindLinesThatContain(params string[] tags);
         void DeleteFile();
     }
 
-    internal class DownloadedLog : IDownloadedLog
+    public class DownloadedLog : IDownloadedLog
     {
         private readonly LogFile logFile;
+        private readonly string owner;
 
-        internal DownloadedLog(LogFile logFile)
+        public DownloadedLog(LogFile logFile, string owner)
         {
             this.logFile = logFile;
+            this.owner = owner;
         }
 
-        public string[] GetLinesContaining(string expectedString)
+        public void AssertLogContains(string expectedString)
         {
             using var file = File.OpenRead(logFile.FullFilename);
             using var streamReader = new StreamReader(file);
-            var lines = new List<string>();
 
             var line = streamReader.ReadLine();
             while (line != null)
             {
-                if (line.Contains(expectedString))
-                {
-                    lines.Add(line);
-                }
+                if (line.Contains(expectedString)) return;
                 line = streamReader.ReadLine();
             }
 
-            return lines.ToArray(); ;
+            Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
         }
 
         public string[] FindLinesThatContain(params string[] tags)

DistTestCore/Logs/LogDownloadHandler.cs

@@ -1,23 +1,25 @@
 using KubernetesWorkflow;
 using Logging;
 
-namespace Core
+namespace DistTestCore.Logs
 {
-    internal class LogDownloadHandler : LogHandler, ILogHandler
+    public class LogDownloadHandler : LogHandler, ILogHandler
     {
+        private readonly RunningContainer container;
         private readonly LogFile log;
 
-        internal LogDownloadHandler(string description, LogFile log)
+        public LogDownloadHandler(RunningContainer container, string description, LogFile log)
         {
+            this.container = container;
             this.log = log;
 
             log.Write($"{description} -->> {log.FullFilename}");
             log.WriteRaw(description);
         }
 
-        internal IDownloadedLog DownloadLog()
+        public DownloadedLog DownloadLog()
         {
-            return new DownloadedLog(log);
+            return new DownloadedLog(log, container.Name);
         }
 
         protected override void ProcessLine(string line)

DistTestCore/Marketplace/CodexContractsContainerConfig.cs

@@ -0,0 +1,16 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class CodexContractsContainerConfig
+    {
+        public CodexContractsContainerConfig(string bootstrapNodeIp, Port jsonRpcPort)
+        {
+            BootstrapNodeIp = bootstrapNodeIp;
+            JsonRpcPort = jsonRpcPort;
+        }
+
+        public string BootstrapNodeIp { get; }
+        public Port JsonRpcPort { get; }
+    }
+}

DistTestCore/Marketplace/CodexContractsContainerRecipe.cs

@@ -0,0 +1,25 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class CodexContractsContainerRecipe : DefaultContainerRecipe
+    {
+        public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
+        public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
+
+        public override string AppName => "codex-contracts";
+        public override string Image => "codexstorage/codex-contracts-eth:latest-dist-tests";
+
+        protected override void InitializeRecipe(StartupConfig startupConfig)
+        {
+            var config = startupConfig.Get<CodexContractsContainerConfig>();
+
+            var ip = config.BootstrapNodeIp;
+            var port = config.JsonRpcPort.Number;
+
+            AddEnvVar("DISTTEST_NETWORK_URL", $"http://{ip}:{port}");
+            AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork");
+            AddEnvVar("KEEP_ALIVE", "1");
+        }
+    }
+}

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -0,0 +1,103 @@
+using KubernetesWorkflow;
+using Utils;
+
+namespace DistTestCore.Marketplace
+{
+    public class CodexContractsStarter : BaseStarter
+    {
+
+        public CodexContractsStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
+        {
+        }
+
+        public MarketplaceInfo Start(GethBootstrapNodeInfo bootstrapNode)
+        {
+            LogStart("Deploying Codex Marketplace...");
+
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]);
+
+            var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig);
+            if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure.");
+            var container = containers.Containers[0];
+
+            WaitUntil(() =>
+            {
+                var logHandler = new ContractsReadyLogHandler(Debug);
+                workflow.DownloadContainerLog(container, logHandler, null);
+                return logHandler.Found;
+            });
+            Log("Contracts deployed. Extracting addresses...");
+
+            var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
+            var marketplaceAddress = extractor.ExtractMarketplaceAddress();
+            var abi = extractor.ExtractMarketplaceAbi();
+
+            var interaction = bootstrapNode.StartInteraction(lifecycle);
+            var tokenAddress = interaction.GetTokenAddress(marketplaceAddress);
+
+            LogEnd("Extract completed. Marketplace deployed.");
+
+            return new MarketplaceInfo(marketplaceAddress, abi, tokenAddress);
+        }
+
+        private void WaitUntil(Func<bool> predicate)
+        {
+            Time.WaitUntil(predicate, TimeSpan.FromMinutes(3), TimeSpan.FromSeconds(2));
+        }
+
+        private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer)
+        {
+            var startupConfig = new StartupConfig();
+            var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.PodInfo.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
+            startupConfig.Add(contractsConfig);
+            return startupConfig;
+        }
+    }
+
+    public class MarketplaceInfo
+    {
+        public MarketplaceInfo(string address, string abi, string tokenAddress)
+        {
+            Address = address;
+            Abi = abi;
+            TokenAddress = tokenAddress;
+        }
+
+        public string Address { get; }
+        public string Abi { get; }
+        public string TokenAddress { get; }
+    }
+
+    public class ContractsReadyLogHandler : LogHandler
+    {
+        // Log should contain 'Compiled 15 Solidity files successfully' at some point.
+        private const string RequiredCompiledString = "Solidity files successfully";
+        // When script is done, it prints the ready-string.
+        private const string ReadyString = "Done! Sleeping indefinitely...";
+        private readonly Action<string> debug;
+
+        public ContractsReadyLogHandler(Action<string> debug)
+        {
+            this.debug = debug;
+            debug($"Looking for '{RequiredCompiledString}' and '{ReadyString}' in container logs...");
+        }
+
+        public bool SeenCompileString { get; private set; }
+        public bool Found { get; private set; }
+
+        protected override void ProcessLine(string line)
+        {
+            debug(line);
+            if (line.Contains(RequiredCompiledString)) SeenCompileString = true;
+            if (line.Contains(ReadyString))
+            {
+                if (!SeenCompileString) throw new Exception("CodexContracts deployment failed. " +
+                    "Solidity files not compiled before process exited.");
+
+                Found = true;
+            }
+        }
+    }
+}

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -1,17 +1,18 @@
 using KubernetesWorkflow;
-using KubernetesWorkflow.Types;
 using Logging;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 using Utils;
 
-namespace GethPlugin
+namespace DistTestCore.Marketplace
 {
-    public class GethContainerInfoExtractor
+    public class ContainerInfoExtractor
     {
-        private readonly ILog log;
-        private readonly IStartupWorkflow workflow;
+        private readonly BaseLog log;
+        private readonly StartupWorkflow workflow;
         private readonly RunningContainer container;
 
-        public GethContainerInfoExtractor(ILog log, IStartupWorkflow workflow, RunningContainer container)
+        public ContainerInfoExtractor(BaseLog log, StartupWorkflow workflow, RunningContainer container)
         {
             this.log = log;
             this.workflow = workflow;
@@ -37,11 +38,45 @@ namespace GethPlugin
             return pubKey;
         }
 
+        public string ExtractMarketplaceAddress()
+        {
+            log.Debug();
+            var marketplaceAddress = Retry(FetchMarketplaceAddress);
+            if (string.IsNullOrEmpty(marketplaceAddress)) throw new InvalidOperationException("Unable to fetch marketplace account from codex-contracts node. Test infra failure.");
+
+            return marketplaceAddress;
+        }
+
+        public string ExtractMarketplaceAbi()
+        {
+            log.Debug();
+            var marketplaceAbi = Retry(FetchMarketplaceAbi);
+            if (string.IsNullOrEmpty(marketplaceAbi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
+
+            return marketplaceAbi;
+        }
+
         private string FetchAccountsCsv()
         {
             return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename);
         }
 
+        private string FetchMarketplaceAddress()
+        {
+            var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceAddressFilename);
+            var marketplace = JsonConvert.DeserializeObject<MarketplaceJson>(json);
+            return marketplace!.address;
+        }
+
+        private string FetchMarketplaceAbi()
+        {
+            var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceArtifactFilename);
+
+            var artifact = JObject.Parse(json);
+            var abi = artifact["abi"];
+            return abi!.ToString(Formatting.None);
+        }
+
         private string FetchPubKey()
         {
             var enodeFinder = new PubKeyFinder(s => log.Debug(s));
@@ -60,7 +95,7 @@ namespace GethPlugin
 
         private static string Retry(Func<string> fetch)
         {
-            return Time.Retry(fetch, nameof(GethContainerInfoExtractor));
+            return Time.Retry(fetch, nameof(ContainerInfoExtractor));
         }
     }
 
@@ -106,4 +141,9 @@ namespace GethPlugin
                     length: closeIndex - openIndex);
         }
     }
+
+    public class MarketplaceJson
+    {
+        public string address { get; set; } = string.Empty;
+    }
 }

DistTestCore/Marketplace/GethBootstrapNodeInfo.cs

@@ -0,0 +1,42 @@
+using KubernetesWorkflow;
+using NethereumWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class GethBootstrapNodeInfo
+    {
+        public GethBootstrapNodeInfo(RunningContainers runningContainers, AllGethAccounts allAccounts, string pubKey, Port discoveryPort)
+        {
+            RunningContainers = runningContainers;
+            AllAccounts = allAccounts;
+            Account = allAccounts.Accounts[0];
+            PubKey = pubKey;
+            DiscoveryPort = discoveryPort;
+        }
+
+        public RunningContainers RunningContainers { get; }
+        public AllGethAccounts AllAccounts { get; }
+        public GethAccount Account { get; }
+        public string PubKey { get; }
+        public Port DiscoveryPort { get; }
+
+        public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
+        {
+            var address = lifecycle.Configuration.GetAddress(RunningContainers.Containers[0]);
+            var account = Account;
+
+            var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, account.PrivateKey);
+            return creator.CreateWorkflow();
+        }
+    }
+
+    public class AllGethAccounts
+    {
+        public GethAccount[] Accounts { get; }
+
+        public AllGethAccounts(GethAccount[] accounts)
+        {
+            Accounts = accounts;
+        }
+    }
+}

DistTestCore/Marketplace/GethBootstrapNodeStarter.cs

@@ -0,0 +1,40 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class GethBootstrapNodeStarter : BaseStarter
+    {
+        public GethBootstrapNodeStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
+        {
+        }
+
+        public GethBootstrapNodeInfo StartGethBootstrapNode()
+        {
+            LogStart("Starting Geth bootstrap node...");
+            var startupConfig = CreateBootstrapStartupConfig();
+            
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
+            if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
+            var bootstrapContainer = containers.Containers[0];
+
+            var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer);
+            var accounts = extractor.ExtractAccounts();
+            var pubKey = extractor.ExtractPubKey();
+            var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
+            var result = new GethBootstrapNodeInfo(containers, accounts, pubKey, discoveryPort);
+
+            LogEnd($"Geth bootstrap node started with account '{result.Account.Account}'");
+
+            return result;
+        }
+
+        private StartupConfig CreateBootstrapStartupConfig()
+        {
+            var config = new StartupConfig();
+            config.Add(new GethStartupConfig(true, null!, 0, 0));
+            return config;
+        }
+    }
+}

DistTestCore/Marketplace/GethCompanionNodeInfo.cs

@@ -0,0 +1,38 @@
+using KubernetesWorkflow;
+using NethereumWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class GethCompanionNodeInfo
+    {
+        public GethCompanionNodeInfo(RunningContainer runningContainer, GethAccount[] accounts)
+        {
+            RunningContainer = runningContainer;
+            Accounts = accounts;
+        }
+
+        public RunningContainer RunningContainer { get; }
+        public GethAccount[] Accounts { get; }
+        
+        public NethereumInteraction StartInteraction(TestLifecycle lifecycle, GethAccount account)
+        {
+            var address = lifecycle.Configuration.GetAddress(RunningContainer);
+            var privateKey = account.PrivateKey;
+
+            var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, privateKey);
+            return creator.CreateWorkflow();
+        }
+    }
+
+    public class GethAccount
+    {
+        public GethAccount(string account, string privateKey)
+        {
+            Account = account;
+            PrivateKey = privateKey;
+        }
+
+        public string Account { get; }
+        public string PrivateKey { get; }
+    }
+}

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -0,0 +1,77 @@
+using KubernetesWorkflow;
+using Utils;
+
+namespace DistTestCore.Marketplace
+{
+    public class GethCompanionNodeStarter : BaseStarter
+    {
+        private int companionAccountIndex = 0;
+
+        public GethCompanionNodeStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
+        {
+        }
+
+        public GethCompanionNodeInfo StartCompanionNodeFor(CodexSetup codexSetup, MarketplaceNetwork marketplace)
+        {
+            LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes.");
+
+            var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
+
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
+            if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
+            var container = containers.Containers[0];
+
+            var node = CreateCompanionInfo(container, marketplace, config);
+            EnsureCompanionNodeIsSynced(node, marketplace);
+
+            LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]");
+            return node;
+        }
+
+        private GethCompanionNodeInfo CreateCompanionInfo(RunningContainer container, MarketplaceNetwork marketplace, GethStartupConfig config)
+        {
+            var accounts = ExtractAccounts(marketplace, config);
+            return new GethCompanionNodeInfo(container, accounts);
+        }
+
+        private static GethAccount[] ExtractAccounts(MarketplaceNetwork marketplace, GethStartupConfig config)
+        {
+            return marketplace.Bootstrap.AllAccounts.Accounts
+                .Skip(1 + config.CompanionAccountStartIndex)
+                .Take(config.NumberOfCompanionAccounts)
+                .ToArray();
+        }
+
+        private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
+        {
+            try
+            {
+                Time.WaitUntil(() =>
+                {
+                    var interaction = node.StartInteraction(lifecycle, node.Accounts.First());
+                    return interaction.IsSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
+                }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
+            }
+            catch (Exception e)
+            {
+                throw new Exception("Geth companion node did not sync within timeout. Test infra failure.", e);
+            }
+        }
+
+        private GethStartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
+        {
+            var config = new GethStartupConfig(false, bootstrapNode, companionAccountIndex, numberOfAccounts);
+            companionAccountIndex += numberOfAccounts;
+            return config;
+        }
+
+        private StartupConfig CreateStartupConfig(GethStartupConfig gethConfig)
+        {
+            var config = new StartupConfig();
+            config.Add(gethConfig);
+            return config;
+        }
+    }
+}

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -0,0 +1,73 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class GethContainerRecipe : DefaultContainerRecipe
+    {
+        private const string defaultArgs = "--ipcdisable --syncmode full";
+
+        public const string HttpPortTag = "http_port";
+        public const string DiscoveryPortTag = "disc_port";
+        public const string AccountsFilename = "accounts.csv";
+
+        public override string AppName => "geth";
+        public override string Image => "codexstorage/dist-tests-geth:latest";
+
+        protected override void InitializeRecipe(StartupConfig startupConfig)
+        {
+            var config = startupConfig.Get<GethStartupConfig>();
+
+            var args = CreateArgs(config);
+
+            AddEnvVar("GETH_ARGS", args);
+        }
+
+        private string CreateArgs(GethStartupConfig config)
+        {
+            var discovery = AddInternalPort(tag: DiscoveryPortTag);
+
+            if (config.IsBootstrapNode)
+            {
+                return CreateBootstapArgs(discovery);
+            }
+
+            return CreateCompanionArgs(discovery, config);
+        }
+
+        private string CreateBootstapArgs(Port discovery)
+        {
+            AddEnvVar("ENABLE_MINER", "1");
+            UnlockAccounts(0, 1);
+            var exposedPort = AddExposedPort(tag: HttpPortTag);
+            return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}";
+        }
+
+        private string CreateCompanionArgs(Port discovery, GethStartupConfig config)
+        {
+            UnlockAccounts(
+                config.CompanionAccountStartIndex + 1,
+                config.NumberOfCompanionAccounts);
+
+            var port = AddInternalPort();
+            var authRpc = AddInternalPort();
+            var httpPort = AddExposedPort(tag: HttpPortTag);
+
+            var bootPubKey = config.BootstrapNode.PubKey;
+            var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.PodInfo.Ip;
+            var bootPort = config.BootstrapNode.DiscoveryPort.Number;
+            var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}";
+
+            return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}";
+        }
+
+        private void UnlockAccounts(int startIndex, int numberOfAccounts)
+        {
+            if (startIndex < 0) throw new ArgumentException();
+            if (numberOfAccounts < 1) throw new ArgumentException();
+            if (startIndex + numberOfAccounts > 1000) throw new ArgumentException("Out of accounts!");
+
+            AddEnvVar("UNLOCK_START_INDEX", startIndex.ToString());
+            AddEnvVar("UNLOCK_NUMBER", numberOfAccounts.ToString());
+        }
+    }
+}

DistTestCore/Marketplace/GethStartResult.cs

@@ -0,0 +1,19 @@
+using Newtonsoft.Json;
+
+namespace DistTestCore.Marketplace
+{
+    public class GethStartResult
+    {
+        public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
+        {
+            MarketplaceAccessFactory = marketplaceAccessFactory;
+            MarketplaceNetwork = marketplaceNetwork;
+            CompanionNode = companionNode;
+        }
+
+        [JsonIgnore]
+        public IMarketplaceAccessFactory MarketplaceAccessFactory { get; }
+        public MarketplaceNetwork MarketplaceNetwork { get; }
+        public GethCompanionNodeInfo CompanionNode { get; }
+    }
+}

DistTestCore/Marketplace/GethStartupConfig.cs

@@ -0,0 +1,18 @@
+namespace DistTestCore.Marketplace
+{
+    public class GethStartupConfig
+    {
+        public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int companionAccountStartIndex, int numberOfCompanionAccounts)
+        {
+            IsBootstrapNode = isBootstrapNode;
+            BootstrapNode = bootstrapNode;
+            CompanionAccountStartIndex = companionAccountStartIndex;
+            NumberOfCompanionAccounts = numberOfCompanionAccounts;
+        }
+
+        public bool IsBootstrapNode { get; }
+        public GethBootstrapNodeInfo BootstrapNode { get; }
+        public int CompanionAccountStartIndex { get; }
+        public int NumberOfCompanionAccounts { get; }
+    }
+}

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -0,0 +1,243 @@
+using DistTestCore.Codex;
+using DistTestCore.Helpers;
+using Logging;
+using Newtonsoft.Json;
+using NUnit.Framework;
+using NUnit.Framework.Constraints;
+using System.Numerics;
+using Utils;
+
+namespace DistTestCore.Marketplace
+{
+    public interface IMarketplaceAccess
+    {
+        string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration);
+        StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
+        void AssertThatBalance(IResolveConstraint constraint, string message = "");
+        TestToken GetBalance();
+    }
+
+    public class MarketplaceAccess : IMarketplaceAccess
+    {
+        private readonly TestLifecycle lifecycle;
+        private readonly MarketplaceNetwork marketplaceNetwork;
+        private readonly GethAccount account;
+        private readonly CodexAccess codexAccess;
+
+        public MarketplaceAccess(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
+        {
+            this.lifecycle = lifecycle;
+            this.marketplaceNetwork = marketplaceNetwork;
+            this.account = account;
+            this.codexAccess = codexAccess;
+        }
+
+        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        {
+            var request = new CodexSalesRequestStorageRequest
+            {
+                duration = ToDecInt(duration.TotalSeconds),
+                proofProbability = ToDecInt(proofProbability),
+                reward = ToDecInt(pricePerSlotPerSecond),
+                collateral = ToDecInt(requiredCollateral),
+                expiry = null,
+                nodes = minRequiredNumberOfNodes,
+                tolerance = null,
+            };
+
+            Log($"Requesting storage for: {contentId.Id}... (" +
+                $"pricePerSlotPerSecond: {pricePerSlotPerSecond}, " +
+                $"requiredCollateral: {requiredCollateral}, " +
+                $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
+                $"proofProbability: {proofProbability}, " +
+                $"duration: {Time.FormatDuration(duration)})");
+
+            var response = codexAccess.RequestStorage(request, contentId.Id);
+
+            if (response == "Purchasing not available")
+            {
+                throw new InvalidOperationException(response);
+            }
+
+            Log($"Storage requested successfully. PurchaseId: '{response}'.");
+
+            return new StoragePurchaseContract(lifecycle.Log, codexAccess, response, duration);
+        }
+
+        public string MakeStorageAvailable(ByteSize totalSpace, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration)
+        {
+            var request = new CodexSalesAvailabilityRequest
+            {
+                size = ToDecInt(totalSpace.SizeInBytes),
+                duration = ToDecInt(maxDuration.TotalSeconds),
+                maxCollateral = ToDecInt(maxCollateral),
+                minPrice = ToDecInt(minPriceForTotalSpace)
+            };
+
+            Log($"Making storage available... (" +
+                $"size: {totalSpace}, " +
+                $"minPriceForTotalSpace: {minPriceForTotalSpace}, " +
+                $"maxCollateral: {maxCollateral}, " +
+                $"maxDuration: {Time.FormatDuration(maxDuration)})");
+
+            var response = codexAccess.SalesAvailability(request);
+
+            Log($"Storage successfully made available. Id: {response.id}");
+
+            return response.id;
+        }
+
+        private string ToDecInt(double d)
+        {
+            var i = new BigInteger(d);
+            return i.ToString("D");
+        }
+
+        public string ToDecInt(TestToken t)
+        {
+            var i = new BigInteger(t.Amount);
+            return i.ToString("D");
+        }
+
+        public void AssertThatBalance(IResolveConstraint constraint, string message = "")
+        {
+            AssertHelpers.RetryAssert(constraint, GetBalance, message);
+        }
+
+        public TestToken GetBalance()
+        {
+            var interaction = marketplaceNetwork.StartInteraction(lifecycle);
+            var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account.Account);
+            var balance = new TestToken(amount);
+
+            Log($"Balance of {account.Account} is {balance}.");
+
+            return balance;
+        }
+
+        private void Log(string msg)
+        {
+            lifecycle.Log.Log($"{codexAccess.Container.Name} {msg}");
+        }
+    }
+
+    public class MarketplaceUnavailable : IMarketplaceAccess
+    {
+        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        {
+            Unavailable();
+            return null!;
+        }
+
+        public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration)
+        {
+            Unavailable();
+            return string.Empty;
+        }
+
+        public void AssertThatBalance(IResolveConstraint constraint, string message = "")
+        {
+            Unavailable();
+        }
+
+        public TestToken GetBalance()
+        {
+            Unavailable();
+            return new TestToken(0);
+        }
+
+        private void Unavailable()
+        {
+            Assert.Fail("Incorrect test setup: Marketplace was not enabled for this group of Codex nodes. Add 'EnableMarketplace(...)' after 'SetupCodexNodes()' to enable it.");
+            throw new InvalidOperationException();
+        }
+    }
+
+    public class StoragePurchaseContract
+    {
+        private readonly BaseLog log;
+        private readonly CodexAccess codexAccess;
+        private DateTime? contractStartUtc;
+
+        public StoragePurchaseContract(BaseLog log, CodexAccess codexAccess, string purchaseId, TimeSpan contractDuration)
+        {
+            this.log = log;
+            this.codexAccess = codexAccess;
+            PurchaseId = purchaseId;
+            ContractDuration = contractDuration;
+        }
+
+        public string PurchaseId { get; }
+        public TimeSpan ContractDuration { get; }
+
+        public void WaitForStorageContractStarted()
+        {
+            WaitForStorageContractStarted(TimeSpan.FromSeconds(30));
+        }
+
+        public void WaitForStorageContractFinished()
+        {
+            if (!contractStartUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var gracePeriod = TimeSpan.FromSeconds(10);
+            var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
+            var timeout = (ContractDuration - currentContractTime) + gracePeriod;
+            WaitForStorageContractState(timeout, "finished");
+        }
+
+        /// <summary>
+        /// Wait for contract to start. Max timeout depends on contract filesize. Allows more time for larger files.
+        /// </summary>
+        public void WaitForStorageContractStarted(ByteSize contractFileSize)
+        {
+            var filesizeInMb = contractFileSize.SizeInBytes / (1024 * 1024);
+            var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0);
+
+            WaitForStorageContractStarted(maxWaitTime);
+        }
+
+        public void WaitForStorageContractStarted(TimeSpan timeout)
+        {
+            WaitForStorageContractState(timeout, "started");
+            contractStartUtc = DateTime.UtcNow;
+        }
+
+        private void WaitForStorageContractState(TimeSpan timeout, string desiredState)
+        {
+            var lastState = "";
+            var waitStart = DateTime.UtcNow;
+            
+            log.Log($"Waiting for {Time.FormatDuration(timeout)} for contract '{PurchaseId}' to reach state '{desiredState}'.");
+            while (lastState != desiredState)
+            {
+                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
+                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
+                if (purchaseStatus != null && purchaseStatus.state != lastState)
+                {
+                    lastState = purchaseStatus.state;
+                    log.Debug("Purchase status: " + statusJson);
+                }
+
+                Thread.Sleep(1000);
+
+                if (lastState == "errored")
+                {
+                    Assert.Fail("Contract errored: " + statusJson);
+                }
+
+                if (DateTime.UtcNow - waitStart > timeout)
+                {
+                    Assert.Fail($"Contract did not reach '{desiredState}' within timeout. {statusJson}");
+                }
+            }
+            log.Log($"Contract '{desiredState}'.");
+        }
+
+        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
+        {
+            return codexAccess.GetPurchaseStatus(purchaseId);
+        }
+    }
+}

DistTestCore/Marketplace/MarketplaceAccessFactory.cs

@@ -0,0 +1,41 @@
+using DistTestCore.Codex;
+
+namespace DistTestCore.Marketplace
+{
+    public interface IMarketplaceAccessFactory
+    {
+        IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access);
+    }
+
+    public class MarketplaceUnavailableAccessFactory : IMarketplaceAccessFactory
+    {
+        public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
+        {
+            return new MarketplaceUnavailable();
+        }
+    }
+
+    public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory
+    {
+        private readonly TestLifecycle lifecycle;
+        private readonly MarketplaceNetwork marketplaceNetwork;
+
+        public GethMarketplaceAccessFactory(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork)
+        {
+            this.lifecycle = lifecycle;
+            this.marketplaceNetwork = marketplaceNetwork;
+        }
+
+        public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
+        {
+            var companionNode = GetGethCompanionNode(access);
+            return new MarketplaceAccess(lifecycle, marketplaceNetwork, companionNode, access);
+        }
+
+        private GethAccount GetGethCompanionNode(CodexAccess access)
+        {
+            var account = access.Container.Recipe.Additionals.Single(a => a is GethAccount);
+            return (GethAccount)account;
+        }
+    }
+}

DistTestCore/Marketplace/MarketplaceInitialConfig.cs

@@ -0,0 +1,17 @@
+namespace DistTestCore.Marketplace
+{
+    public class MarketplaceInitialConfig
+    {
+        public MarketplaceInitialConfig(Ether initialEth, TestToken initialTestTokens, bool isValidator)
+        {
+            InitialEth = initialEth;
+            InitialTestTokens = initialTestTokens;
+            IsValidator = isValidator;
+        }
+
+        public Ether InitialEth { get; }
+        public TestToken InitialTestTokens { get; }
+        public bool IsValidator { get; }
+        public int? AccountIndexOverride { get; set; }
+    }
+}

DistTestCore/Marketplace/MarketplaceNetwork.cs

@@ -0,0 +1,21 @@
+using NethereumWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class MarketplaceNetwork
+    {
+        public MarketplaceNetwork(GethBootstrapNodeInfo bootstrap, MarketplaceInfo marketplace)
+        {
+            Bootstrap = bootstrap;
+            Marketplace = marketplace;
+        }
+
+        public GethBootstrapNodeInfo Bootstrap { get; }
+        public MarketplaceInfo Marketplace { get; }
+
+        public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
+        {
+            return Bootstrap.StartInteraction(lifecycle);
+        }
+    }
+}

DistTestCore/Metrics/GrafanaContainerRecipe.cs

@@ -0,0 +1,25 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Metrics
+{
+    public class GrafanaContainerRecipe : DefaultContainerRecipe
+    {
+        public override string AppName => "grafana";
+        public override string Image => "grafana/grafana-oss:10.0.3";
+
+        public const string DefaultAdminUser = "adminium";
+        public const string DefaultAdminPassword = "passwordium";
+
+        protected override void InitializeRecipe(StartupConfig startupConfig)
+        {
+            AddExposedPort(3000);
+
+            AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true");
+            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org.");
+            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor");
+
+            AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser);
+            AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword);
+        }
+    }
+}

DistTestCore/Metrics/MetricsAccess.cs

@@ -0,0 +1,81 @@
+using DistTestCore.Helpers;
+using KubernetesWorkflow;
+using Logging;
+using NUnit.Framework;
+using NUnit.Framework.Constraints;
+using Utils;
+
+namespace DistTestCore.Metrics
+{
+    public interface IMetricsAccess
+    {
+        void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
+    }
+
+    public class MetricsAccess : IMetricsAccess
+    {
+        private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
+        private readonly MetricsQuery query;
+        private readonly RunningContainer node;
+
+        public MetricsAccess(BaseLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
+        {
+            this.log = log;
+            this.timeSet = timeSet;
+            this.query = query;
+            this.node = node;
+        }
+
+        public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
+        {
+            AssertHelpers.RetryAssert(constraint, () =>
+            {
+                var metricSet = GetMetricWithTimeout(metricName);
+                var metricValue = metricSet.Values[0].Value;
+
+                log.Log($"{node.Name} metric '{metricName}' = {metricValue}");
+                return metricValue;
+            }, message);
+        }
+
+        public Metrics? GetAllMetrics()
+        {
+            return query.GetAllMetricsForNode(node);
+        }
+
+        private MetricsSet GetMetricWithTimeout(string metricName)
+        {
+            var start = DateTime.UtcNow;
+
+            while (true)
+            {
+                var mostRecent = GetMostRecent(metricName);
+                if (mostRecent != null) return mostRecent;
+                if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout())
+                {
+                    Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
+                    throw new TimeoutException();
+                }
+
+                Time.Sleep(TimeSpan.FromSeconds(2));
+            }
+        }
+
+        private MetricsSet? GetMostRecent(string metricName)
+        {
+            var result = query.GetMostRecent(metricName, node);
+            if (result == null) return null;
+            return result.Sets.LastOrDefault();
+        }
+    }
+
+    public class MetricsUnavailable : IMetricsAccess
+    {
+        public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
+        {
+            Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
+            throw new InvalidOperationException();
+        }
+    }
+}

DistTestCore/Metrics/MetricsAccessFactory.cs

@@ -0,0 +1,35 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Metrics
+{
+    public interface IMetricsAccessFactory
+    {
+        IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer);
+    }
+
+    public class MetricsUnavailableAccessFactory : IMetricsAccessFactory
+    {
+        public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
+        {
+            return new MetricsUnavailable();
+        }
+    }
+
+    public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory
+    {
+        private readonly TestLifecycle lifecycle;
+        private readonly RunningContainers prometheusContainer;
+
+        public CodexNodeMetricsAccessFactory(TestLifecycle lifecycle, RunningContainers prometheusContainer)
+        {
+            this.lifecycle = lifecycle;
+            this.prometheusContainer = prometheusContainer;
+        }
+
+        public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
+        {
+            var query = new MetricsQuery(lifecycle, prometheusContainer);
+            return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
+        }
+    }
+}

DistTestCore/Metrics/MetricsDownloader.cs

@@ -1,29 +1,29 @@
 using Logging;
 using System.Globalization;
 
-namespace MetricsPlugin
+namespace DistTestCore.Metrics
 {
     public class MetricsDownloader
     {
-        private readonly ILog log;
+        private readonly BaseLog log;
 
-        public MetricsDownloader(ILog log)
+        public MetricsDownloader(BaseLog log)
         {
             this.log = log;
         }
 
-        public LogFile? DownloadAllMetrics(string targetName, IMetricsAccess access)
+        public void DownloadAllMetricsForNode(string nodeName, MetricsAccess access)
         {
             var metrics = access.GetAllMetrics();
-            if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return null;
+            if (metrics == null || metrics.Sets.Length == 0 || metrics.Sets.All(s => s.Values.Length == 0)) return;
 
             var headers = new[] { "timestamp" }.Concat(metrics.Sets.Select(s => s.Name)).ToArray();
             var map = CreateValueMap(metrics);
 
-            return WriteToFile(targetName, headers, map);
+            WriteToFile(nodeName, headers, map);
         }
 
-        private LogFile WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)
+        private void WriteToFile(string nodeName, string[] headers, Dictionary<DateTime, List<string>> map)
         {
             var file = log.CreateSubfile("csv");
             log.Log($"Downloading metrics for {nodeName} to file {file.FullFilename}");
@@ -34,8 +34,6 @@ namespace MetricsPlugin
             {
                 file.WriteRaw(string.Join(",", new[] { FormatTimestamp(pair.Key) }.Concat(pair.Value)));
             }
-
-            return file;
         }
 
         private Dictionary<DateTime, List<string>> CreateValueMap(Metrics metrics)

DistTestCore/Metrics/MetricsMode.cs

@@ -0,0 +1,9 @@
+namespace DistTestCore.Metrics
+{
+    public enum MetricsMode
+    {
+        None,
+        Record,
+        Dashboard
+    }
+}

DistTestCore/Metrics/MetricsQuery.cs

@@ -1,30 +1,34 @@
-using Core;
-using KubernetesWorkflow.Types;
-using Logging;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
 using System.Globalization;
 
-namespace MetricsPlugin
+namespace DistTestCore.Metrics
 {
     public class MetricsQuery
     {
-        private readonly IHttp http;
-        private readonly ILog log;
+        private readonly Http http;
 
-        public MetricsQuery(IPluginTools tools, RunningContainer runningContainer)
+        public MetricsQuery(TestLifecycle lifecycle, RunningContainers runningContainers)
         {
-            RunningContainer = runningContainer;
-            log = tools.GetLog();
-            http = tools.CreateHttp(RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag), "api/v1");
+            RunningContainers = runningContainers;
+
+            var address = lifecycle.Configuration.GetAddress(runningContainers.Containers[0]);
+
+            http = new Http(
+                lifecycle.Log,
+                lifecycle.TimeSet,
+                address,
+                "api/v1");
         }
 
-        public RunningContainer RunningContainer { get; }
+        public RunningContainers RunningContainers { get; }
 
-        public Metrics? GetMostRecent(string metricName, IMetricsScrapeTarget target)
+        public Metrics? GetMostRecent(string metricName, RunningContainer node)
         {
-            var response = GetLastOverTime(metricName, GetInstanceStringForNode(target));
+            var response = GetLastOverTime(metricName, GetInstanceStringForNode(node));
             if (response == null) return null;
 
-            var result = new Metrics
+            return new Metrics
             {
                 Sets = response.data.result.Select(r =>
                 {
@@ -35,27 +39,20 @@ namespace MetricsPlugin
                     };
                 }).ToArray()
             };
-
-            Log(target, metricName, result);
-            return result;
         }
 
         public Metrics? GetMetrics(string metricName)
         {
             var response = GetAll(metricName);
             if (response == null) return null;
-            var result = MapResponseToMetrics(response);
-            Log(metricName, result);
-            return result;
+            return MapResponseToMetrics(response);
         }
 
-        public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target)
+        public Metrics? GetAllMetricsForNode(RunningContainer node)
         {
-            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}");
+            var response = http.HttpGetJson<PrometheusQueryResponse>($"query?query={GetInstanceStringForNode(node)}{GetQueryTimeRange()}");
             if (response.status != "success") return null;
-            var result = MapResponseToMetrics(response);
-            Log(target, result);
-            return result;
+            return MapResponseToMetrics(response);
         }
 
         private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString)
@@ -120,14 +117,16 @@ namespace MetricsPlugin
             };
         }
 
-        private string GetInstanceNameForNode(IMetricsScrapeTarget target)
+        private string GetInstanceNameForNode(RunningContainer node)
         {
-            return ScrapeTargetHelper.FormatTarget(log, target);
+            var ip = node.Pod.PodInfo.Ip;
+            var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
+            return $"{ip}:{port}";
         }
 
-        private string GetInstanceStringForNode(IMetricsScrapeTarget target)
+        private string GetInstanceStringForNode(RunningContainer node)
         {
-            return "{instance=\"" + GetInstanceNameForNode(target) + "\"}";
+            return "{instance=\"" + GetInstanceNameForNode(node) + "\"}";
         }
 
         private string GetQueryTimeRange()
@@ -145,36 +144,11 @@ namespace MetricsPlugin
             var unixSeconds = ToValue(v);
             return new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixSeconds);
         }
-
-        private void Log(IMetricsScrapeTarget target, string metricName, Metrics result)
-        {
-            Log($"{target.Container.Name} '{metricName}' = {result}");
-        }
-
-        private void Log(string metricName, Metrics result)
-        {
-            Log($"'{metricName}' = {result}");
-        }
-
-        private void Log(IMetricsScrapeTarget target, Metrics result)
-        {
-            Log($"{target.Container.Name} => {result}");
-        }
-
-        private void Log(string msg)
-        {
-            log.Log(msg);
-        }
     }
 
     public class Metrics
     {
         public MetricsSet[] Sets { get; set; } = Array.Empty<MetricsSet>();
-
-        public override string ToString()
-        {
-            return "[" + string.Join(',', Sets.Select(s => s.ToString())) + "]";
-        }
     }
 
     public class MetricsSet
@@ -182,22 +156,12 @@ namespace MetricsPlugin
         public string Name { get; set; } = string.Empty;
         public string Instance { get; set; } = string.Empty;
         public MetricsSetValue[] Values { get; set; } = Array.Empty<MetricsSetValue>();
-
-        public override string ToString()
-        {
-            return $"{Name} ({Instance}) : {{{string.Join(",", Values.Select(v => v.ToString()))}}}";
-        }
     }
 
     public class MetricsSetValue
     {
         public DateTime Timestamp { get; set; }
         public double Value { get; set; }
-
-        public override string ToString()
-        {
-            return $"<{Timestamp.ToString("o")}={Value}>";
-        }
     }
 
     public class PrometheusQueryResponse