Removes log asserts, they are unreliable in the cluster.

# Conflicts:
#	DistTestCore/Codex/CodexContainerRecipe.cs

.github/workflows/dist-tests.yaml

@@ -0,0 +1,74 @@
+name: Dist Tests
+
+
+on:
+  # push:
+  #   branches:
+  #     - master
+  #   tags:
+  #     - 'v*.*.*'
+  #   paths-ignore:
+  #     - '**/*.md'
+  #     - '.gitignore'
+  #     - 'docker/**'
+  #     - '!docker/job.yaml'
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: Branch (master)
+        required: false
+        type: string
+      source:
+        description: Repository with tests (current)
+        required: false
+        type: string
+      nameprefix:
+        description: Runner prefix (cs-codex-dist-tests)
+        required: false
+        type: string
+      namespace:
+        description: Runner namespace (cs-codex-dist-tests)
+        required: false
+        type: string
+
+
+env:
+  BRANCH: ${{ github.ref_name }}
+  SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
+  NAMEPREFIX: cs-codex-dist-tests
+  NAMESPACE: cs-codex-dist-tests
+  JOB_MANIFEST: docker/job.yaml
+  KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }}
+  KUBE_VERSION: v1.26.1
+
+
+jobs:
+  run_tests:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Variables
+        run: |
+          [[ -n "${{ inputs.branch }}" ]] && echo "BRANCH=${{ inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
+          [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      - name: Kubectl - Install ${{ env.KUBE_VERSION }}
+        uses: azure/setup-kubectl@v3
+        with:
+          version: ${{ env.KUBE_VERSION }}
+
+      - name: Kubectl - Kubeconfig
+        run: |
+          mkdir -p "${HOME}"/.kube
+          echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
+      - name: Kubectl - Create Job
+        run: |
+          envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -

.github/workflows/docker-reusable.yml

@@ -0,0 +1,177 @@
+name: Docker - Reusable
+
+
+on:
+  workflow_call:
+    inputs:
+      docker_file:
+        default: docker/Dockerfile
+        description: Dockerfile
+        required: false
+        type: string
+      docker_repo:
+        default: codexstorage/cs-codex-dist-tests
+        description: DockerHub repository
+        required: false
+        type: string
+      tag_latest:
+        default: true
+        description: Set latest tag for Docker images
+        required: false
+        type: boolean
+      tag_sha:
+        default: true
+        description: Set Git short commit as Docker tag
+        required: false
+        type: boolean
+      tag_suffix:
+        default: ''
+        description: Suffix for Docker images tag
+        required: false
+        type: string
+
+
+env:
+  DOCKER_FILE: ${{ inputs.docker_file }}
+  DOCKER_REPO: ${{ inputs.docker_repo }}
+  TAG_LATEST: ${{ inputs.tag_latest }}
+  TAG_SHA: ${{ inputs.tag_sha }}
+  TAG_SUFFIX: ${{ inputs.tag_suffix }}
+
+
+jobs:
+  # Build platform specific image
+  build:
+    strategy:
+      fail-fast: true
+      matrix:
+        target:
+          - os: linux
+            arch: amd64
+          - os: linux
+            arch: arm64
+        include:
+          - target:
+              os: linux
+              arch: amd64
+            builder: ubuntu-22.04
+          - target:
+              os: linux
+              arch: arm64
+            builder: buildjet-4vcpu-ubuntu-2204-arm
+
+    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
+    runs-on: ${{ matrix.builder }}
+    env:
+      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_REPO }}
+
+      - name: Docker - Set up Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Build and Push by digest
+        id: build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ${{ env.DOCKER_FILE }}
+          platforms: ${{ env.PLATFORM }}
+          push: true
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Docker - Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Docker - Upload digest
+        uses: actions/upload-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+
+  # Publish multi-platform image
+  publish:
+    name: Publish multi-platform image
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Docker - Variables
+        run: |
+          # Adjust custom suffix when set and
+          if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
+            echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
+          fi
+          # Disable SHA tags on tagged release
+          if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
+            echo "TAG_SHA=false" >>$GITHUB_ENV
+          fi
+          # Handle latest and latest-custom using raw
+          if [[ ${{ env.TAG_SHA }} == "false" ]]; then
+            echo "TAG_LATEST=false" >>$GITHUB_ENV
+            echo "TAG_RAW=true" >>$GITHUB_ENV
+            if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
+              echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
+            else
+              echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
+            fi
+          else
+            echo "TAG_RAW=false" >>$GITHUB_ENV
+          fi
+
+      - name: Docker - Download digests
+        uses: actions/download-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests
+
+      - name: Docker - Set up Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_REPO }}
+          flavor: |
+            latest=${{ env.TAG_LATEST }}
+            suffix=${{ env.TAG_SUFFIX }},onlatest=true
+          tags: |
+            type=semver,pattern={{version}}
+            type=raw,enable=${{ env.TAG_RAW }},value=latest
+            type=sha,enable=${{ env.TAG_SHA }}
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
+
+      - name: Docker - Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}

.github/workflows/docker.yml

@@ -0,0 +1,22 @@
+name: Docker
+
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*.*.*'
+    paths:
+      - docker/Dockerfile
+      - docker/docker-entrypoint.sh
+      - .github/workflows/docker.yml
+      - .github/workflows/docker-reusable.yml
+  workflow_dispatch:
+
+
+jobs:
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    secrets: inherit

.gitignore

@@ -1,3 +1,4 @@
 .vs
 obj
 bin
+.vscode

ArgsUniform/ArgsUniform.cs

@@ -0,0 +1,254 @@
+using System.Reflection;
+
+namespace ArgsUniform
+{
+    public class ArgsUniform<T>
+    {
+        private readonly Action printAppInfo;
+        private readonly object? defaultsProvider;
+        private readonly IEnv.IEnv env;
+        private readonly string[] args;
+        private const int cliStart = 8;
+        private const int shortStart = 38;
+        private const int envStart = 48;
+        private const int descStart = 80;
+
+        public ArgsUniform(Action printAppInfo, params string[] args)
+            : this(printAppInfo, new IEnv.Env(), args)
+        {
+        }
+
+        public ArgsUniform(Action printAppInfo, object defaultsProvider, params string[] args)
+            : this(printAppInfo, defaultsProvider, new IEnv.Env(), args)
+        {
+        }
+
+        public ArgsUniform(Action printAppInfo, IEnv.IEnv env, params string[] args)
+            : this(printAppInfo, null!, env, args)
+        {
+        }
+
+        public ArgsUniform(Action printAppInfo, object defaultsProvider, IEnv.IEnv env, params string[] args)
+        {
+            this.printAppInfo = printAppInfo;
+            this.defaultsProvider = defaultsProvider;
+            this.env = env;
+            this.args = args;
+        }
+
+        public T Parse(bool printResult = false)
+        {
+            if (args.Any(a => a == "-h" || a == "--help" || a == "-?"))
+            {
+                printAppInfo();
+                PrintHelp();
+                throw new Exception();
+            }
+
+            var result = Activator.CreateInstance<T>();
+            var uniformProperties = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).ToArray();
+            var missingRequired = new List<PropertyInfo>();
+            foreach (var uniformProperty in uniformProperties)
+            {
+                var attr = uniformProperty.GetCustomAttribute<UniformAttribute>();
+                if (attr != null)
+                {
+                    if (!UniformAssign(result, attr, uniformProperty) && attr.Required)
+                    {
+                        {
+                            missingRequired.Add(uniformProperty);
+                        }
+                    }
+                }
+            }
+
+            if (missingRequired.Any())
+            {
+                PrintResults(result, uniformProperties);
+                Print("");
+                foreach (var missing in missingRequired)
+                {
+                    var attr = missing.GetCustomAttribute<UniformAttribute>()!;
+                    var exampleArg = $"--{attr.Arg}=...";
+                    var exampleEnvVar = $"{attr.EnvVar}=...";
+                    Print($" ! Missing required input. Use argument: '{exampleArg}' or environment variable: '{exampleEnvVar}'.");
+                }
+
+                PrintHelp();
+                throw new ArgumentException("Unable to assemble all required arguments");
+            }
+
+            if (printResult)
+            {
+                PrintResults(result, uniformProperties);
+            }
+
+            return result;
+        }
+
+        private void PrintResults(T result, PropertyInfo[] uniformProperties)
+        {
+            Print("");
+            foreach (var p in uniformProperties)
+            {
+                Print($"\t{p.Name} = {p.GetValue(result)}");
+            }
+            Print("");
+        }
+
+        public void PrintHelp()
+        {
+            Print("");
+            PrintAligned("CLI option:", "(short)", "Environment variable:", "Description");
+            var attrs = typeof(T).GetProperties().Where(m => m.GetCustomAttributes(typeof(UniformAttribute), false).Length == 1).Select(p => p.GetCustomAttribute<UniformAttribute>()).Where(a => a != null).ToArray();
+            foreach (var attr in attrs)
+            {
+                var a = attr!;
+                var optional = !a.Required ? " *" : "";
+                PrintAligned($"--{a.Arg}=...", $"({a.ArgShort})", a.EnvVar, a.Description + optional);
+            }
+            Print("");
+        }
+
+        private void Print(string msg)
+        {
+            Console.WriteLine(msg);
+        }
+
+        private void PrintAligned(string cli, string s, string env, string desc)
+        {
+            Console.CursorLeft = cliStart;
+            Console.Write(cli);
+            Console.CursorLeft = shortStart;
+            Console.Write(s);
+            Console.CursorLeft = envStart;
+            Console.Write(env);
+            Console.CursorLeft = descStart;
+            Console.Write(desc + Environment.NewLine);
+        }
+
+        private object GetDefaultValue(Type t)
+        {
+            if (t.IsValueType) return Activator.CreateInstance(t)!;
+            return null!;
+        }
+
+        private bool UniformAssign(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            if (AssignFromArgsIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromEnvVarIfAble(result, attr, uniformProperty)) return true;
+            if (AssignFromDefaultsIfAble(result, uniformProperty)) return true;
+            return false;
+        }
+
+        private bool AssignFromDefaultsIfAble(T result, PropertyInfo uniformProperty)
+        {
+            var currentValue = uniformProperty.GetValue(result);
+            var isEmptryString = (currentValue as string) == string.Empty;
+            if (currentValue != GetDefaultValue(uniformProperty.PropertyType) && !isEmptryString) return true;
+
+            if (defaultsProvider == null) return false;
+
+            var defaultProperty = defaultsProvider.GetType().GetProperties().SingleOrDefault(p => p.Name == uniformProperty.Name);
+            if (defaultProperty == null) return false;
+
+            var value = defaultProperty.GetValue(defaultsProvider);
+            if (value != null)
+            {
+                return Assign(result, uniformProperty, value);
+            }
+            return false;
+        }
+
+        private bool AssignFromEnvVarIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var e = env.GetEnvVarOrDefault(attr.EnvVar, string.Empty);
+            if (!string.IsNullOrEmpty(e))
+            {
+                return Assign(result, uniformProperty, e);
+            }
+            return false;
+        }
+
+        private bool AssignFromArgsIfAble(T result, UniformAttribute attr, PropertyInfo uniformProperty)
+        {
+            var fromArg = GetFromArgs(attr.Arg);
+            if (fromArg != null)
+            {
+                return Assign(result, uniformProperty, fromArg);
+            }
+            var fromShort = GetFromArgs(attr.ArgShort);
+            if (fromShort != null)
+            {
+                return Assign(result, uniformProperty, fromShort);
+            }
+            return false;
+        }
+
+        private bool Assign(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (uniformProperty.PropertyType == value.GetType())
+            {
+                uniformProperty.SetValue(result, value);
+                return true;
+            }
+            else
+            {
+                if (uniformProperty.PropertyType == typeof(string) || uniformProperty.PropertyType == typeof(int))
+                {
+                    uniformProperty.SetValue(result, Convert.ChangeType(value, uniformProperty.PropertyType));
+                    return true;
+                }
+                else
+                {
+                    if (uniformProperty.PropertyType == typeof(int?)) return AssignOptionalInt(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType.IsEnum) return AssignEnum(result, uniformProperty, value);
+                    if (uniformProperty.PropertyType == typeof(bool)) return AssignBool(result, uniformProperty, value);
+                    
+                    throw new NotSupportedException();
+                }
+            }
+        }
+
+        private static bool AssignEnum(T result, PropertyInfo uniformProperty, object value)
+        {
+            var s = value.ToString();
+            if (Enum.TryParse(uniformProperty.PropertyType, s, out var e))
+            {
+                uniformProperty.SetValue(result, e);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignOptionalInt(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (int.TryParse(value.ToString(), out int i))
+            {
+                uniformProperty.SetValue(result, i);
+                return true;
+            }
+            return false;
+        }
+
+        private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
+        {
+            if (value != null)
+            {
+                uniformProperty.SetValue(result, true);
+            }
+            return true;
+        }
+
+        private string? GetFromArgs(string key)
+        {
+            var argKey = $"--{key}=";
+            var arg = args.FirstOrDefault(a => a.StartsWith(argKey));
+            if (arg != null)
+            {
+                return arg.Substring(argKey.Length);
+            }
+            return null;
+        }
+    }
+}

ArgsUniform/ArgsUniform.csproj

@@ -0,0 +1,13 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="I-Env" Version="1.0.0" />
+  </ItemGroup>
+
+</Project>

ArgsUniform/ExampleUser.cs

@@ -0,0 +1,34 @@
+namespace ArgsUniform
+{
+    public class ExampleUser
+    {
+        public class Args
+        {
+            [Uniform("aaa", "a", "AAA", false, "Sets the AAA!")]
+            public string Aaa { get; set; } = string.Empty;
+
+            [Uniform("bbb", "b", "BBB", true, "Sets that BBB")]
+            public string Bbb { get; set; } = string.Empty;
+        }
+
+        public class DefaultsProvider
+        {
+            public string Aaa { get { return "non-static operation"; } }
+        }
+
+        public void Example()
+        {
+            // env var: "AAA=BBB"
+            var args = "--ccc=ddd";
+
+            var uniform = new ArgsUniform<Args>(PrintHelp, new DefaultsProvider(), args);
+
+            var aaa = uniform.Parse();
+        }
+
+        private static void PrintHelp()
+        {
+            Console.WriteLine("Help text!");
+        }
+    }
+}

ArgsUniform/UniformAttribute.cs

@@ -0,0 +1,20 @@
+namespace ArgsUniform
+{
+    public class UniformAttribute : Attribute
+    {
+        public UniformAttribute(string arg, string argShort, string envVar, bool required, string description)
+        {
+            Arg = arg;
+            ArgShort = argShort;
+            EnvVar = envVar;
+            Required = required;
+            Description = description;
+        }
+
+        public string Arg { get; }
+        public string ArgShort { get; }
+        public string EnvVar { get; }
+        public bool Required { get; }
+        public string Description { get; }
+    }
+}

CONTRIBUTINGTESTS.MD

@@ -0,0 +1,24 @@
+# Distributed System Tests for Nim-Codex
+
+## Contributing tests
+Do you want to write some tests for Codex using this distributed test setup? Great! Here's what you do.
+
+1. Create a branch. Name it something descriptive, but start it with `tests/` please. [Example: `tests/data-redundancy`.]
+1. Checkout your branch, and decide if your tests will be 'short' tests (minutes to hours), or 'long' tests (hours to days), or both! Create a folder for your tests in the matching folders (`/Tests`, `/LongTests`) and don't worry: You can always move your tests later if you like. [Example, short: `/Tests/DataRedundancy/`, long: `/LongTests/DataRedundancy/`]
+1. Create one or more code files in your new folder, and write some tests! Here are some tips to help you get started. You can always take a look at the example tests found in [`/Tests/BasicTests/ExampleTests.cs`](/Tests/BasicTests/ExampleTests.cs)
+   1. Set up a standard NUnit test fixture.
+   1. Inherrit from `DistTest` or `AutoBootstrapDistTest`.
+   1. When using `DistTest`:
+      1. You must start your own Codex bootstrap node. You can use `SetupCodexBootstrapNode(...)` for this.
+      1. When you start other Codex nodes with `SetupCodexNodes(...)` you can pass the bootstrap node by adding the `.WithBootstrapNode(...)` option.
+   1. When using `AutoBootstrapDistTest`:
+      1. The test-infra creates the bootstrap node for you, and automatically passes it to each Codex node you create in your tests. Handy for keeping your tests clean and to-the-point.
+      1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `DistTest` instead.
+   1. You can generate files of random test data by calling `GenerateTestFile(...)`.
+   1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files.
+   1. You can enable access to the Codex node metrics by adding the option `.EnableMetrics()`. Enabling metrics will make the test-infra download and save all Codex metrics in case of a test failure. (The metrics are stored as CSV, in the same location as the test log file.)
+   1. You can enable access to the blockchain marketplace by adding the option `.EnableMarketplace(...)`.
+   1. Enabling metrics and/or enabling the marketplace takes extra resources from the test-infra and increases the time needed during Codex node setup. Please don't enable these features unless your tests need them.
+   1. Tip: Codex nodes can be named. Use the option `WithName(...)` and make reading your test logs a little nicer!
+   1. Tip: Commit often.
+1. Once you're happy with your tests, please create a pull-request and ask (another) Codex core contributor to review your changes.

CodexNetDeployer/CodexNetDeployer.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
+    <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+  </ItemGroup>
+
+</Project>

CodexNetDeployer/CodexNodeStarter.cs

@@ -0,0 +1,119 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using KubernetesWorkflow;
+
+namespace CodexNetDeployer
+{
+    public class CodexNodeStarter
+    {
+        private readonly Configuration config;
+        private readonly TestLifecycle lifecycle;
+        private readonly GethStartResult gethResult;
+        private string bootstrapSpr = "";
+        private int validatorsLeft;
+
+        public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
+        {
+            this.config = config;
+            this.lifecycle = lifecycle;
+            this.gethResult = gethResult;
+            validatorsLeft = numberOfValidators;
+        }
+
+        public CodexNodeStartResult? Start(int i)
+        {
+            Console.Write($" - {i} = ");
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var workflowStartup = new StartupConfig();
+            workflowStartup.Add(gethResult);
+            workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));
+
+            var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
+
+            var container = containers.Containers.First();
+            var codexAccess = new CodexAccess(lifecycle.Log, container, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(container));
+            var account = gethResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[i];
+            var tokenAddress = gethResult.MarketplaceNetwork.Marketplace.TokenAddress;
+            var marketAccess = new MarketplaceAccess(lifecycle, gethResult.MarketplaceNetwork, account, codexAccess);
+
+            try
+            {
+                var debugInfo = codexAccess.GetDebugInfo();
+                if (!string.IsNullOrWhiteSpace(debugInfo.spr))
+                {
+                    Console.Write("Online\t");
+
+                    var interaction = gethResult.MarketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
+                    interaction.MintTestTokens(new[] { account.Account }, config.InitialTestTokens, tokenAddress);
+                    Console.Write("Tokens minted\t");
+
+                    var response = marketAccess.MakeStorageAvailable(
+                        totalSpace: config.StorageSell!.Value.MB(),
+                        minPriceForTotalSpace: config.MinPrice.TestTokens(),
+                        maxCollateral: config.MaxCollateral.TestTokens(),
+                        maxDuration: TimeSpan.FromSeconds(config.MaxDuration));
+
+                    if (!string.IsNullOrEmpty(response))
+                    {
+                        Console.Write("Storage available\tOK" + Environment.NewLine);
+
+                        if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
+                        validatorsLeft--;
+                        return new CodexNodeStartResult(workflow, container, codexAccess);
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine("Exception:" + ex.ToString());
+            }
+
+            Console.Write("Unknown failure. Downloading container log." + Environment.NewLine);
+            lifecycle.DownloadLog(container);
+
+            return null;
+        }
+
+        private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft)
+        {
+            var codexStart = new CodexStartupConfig(config.CodexLogLevel);
+
+            if (!string.IsNullOrEmpty(bootstrapSpr)) codexStart.BootstrapSpr = bootstrapSpr;
+            codexStart.StorageQuota = config.StorageQuota!.Value.MB();
+            var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
+            marketplaceConfig.AccountIndexOverride = i;
+            codexStart.MarketplaceConfig = marketplaceConfig;
+            codexStart.MetricsMode = config.Metrics;
+
+            if (config.BlockTTL != Configuration.SecondsIn1Day)
+            {
+                codexStart.BlockTTL = config.BlockTTL;
+            }
+            if (config.BlockMI != Configuration.TenMinutes)
+            {
+                codexStart.BlockMaintenanceInterval = TimeSpan.FromSeconds(config.BlockMI);
+            }
+            if (config.BlockMN != 1000)
+            {
+                codexStart.BlockMaintenanceNumber = config.BlockMN;
+            }
+
+            return codexStart;
+        }
+    }
+
+    public class CodexNodeStartResult
+    {
+        public CodexNodeStartResult(StartupWorkflow workflow, RunningContainer container, CodexAccess access)
+        {
+            Workflow = workflow;
+            Container = container;
+            Access = access;
+        }
+
+        public StartupWorkflow Workflow { get; }
+        public RunningContainer Container { get; }
+        public CodexAccess Access { get; }
+    }
+}

CodexNetDeployer/Configuration.cs

@@ -0,0 +1,111 @@
+using ArgsUniform;
+using DistTestCore.Codex;
+using DistTestCore.Metrics;
+
+namespace CodexNetDeployer
+{
+    public class Configuration
+    {
+        public const int SecondsIn1Day = 24 * 60 * 60;
+        public const int TenMinutes = 10 * 60;
+
+        [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
+        public string KubeConfigFile { get; set; } = "null";
+
+        [Uniform("kube-namespace", "kn", "KUBENAMESPACE", true, "Kubernetes namespace to be used for deployment.")]
+        public string KubeNamespace { get; set; } = string.Empty;
+
+        [Uniform("nodes", "n", "NODES", true, "Number of Codex nodes to be created.")]
+        public int? NumberOfCodexNodes { get; set; }
+
+        [Uniform("validators", "v", "VALIDATORS", true, "Number of Codex nodes that will be validating.")]
+        public int? NumberOfValidators { get; set; }
+
+        [Uniform("storage-quota", "sq", "STORAGEQUOTA", true, "Storage quota in megabytes used by each Codex node.")]
+        public int? StorageQuota { get; set; }
+
+        [Uniform("storage-sell", "ss", "STORAGESELL", true, "Number of megabytes of storage quota to make available for selling.")]
+        public int? StorageSell { get; set; }
+
+        [Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")]
+        public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug;
+
+        [Uniform("test-tokens", "tt", "TESTTOKENS", true, "Initial amount of test-tokens minted for each Codex node.")]
+        public int InitialTestTokens { get; set; } = int.MaxValue;
+
+        [Uniform("min-price", "mp", "MINPRICE", true, "Minimum price for the storage space for which contracts will be accepted.")]
+        public int MinPrice { get; set; }
+
+        [Uniform("max-collateral", "mc", "MAXCOLLATERAL", true, "Maximum collateral that will be placed for the total storage space.")]
+        public int MaxCollateral { get; set; }
+
+        [Uniform("max-duration", "md", "MAXDURATION", true, "Maximum duration in seconds for contracts which will be accepted.")]
+        public int MaxDuration { get; set; }
+
+        [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
+        public int BlockTTL { get; set; } = SecondsIn1Day;
+
+        [Uniform("block-mi", "bmi", "BLOCKMI", false, "Block maintenance interval in seconds. Default is 10 minutes.")]
+        public int BlockMI { get; set; } = TenMinutes;
+
+        [Uniform("block-mn", "bmn", "BLOCKMN", false, "Number of blocks maintained per interval. Default is 1000 blocks.")]
+        public int BlockMN { get; set; } = 1000;
+
+        [Uniform("metrics", "m", "METRICS", false, "[None*, Record, Dashboard]. Determines if metrics will be recorded and if a dashboard service will be created.")]
+        public MetricsMode Metrics { get; set; } = MetricsMode.None;
+
+        [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
+            "set this option to override the label value.")]
+        public string TestsTypePodLabel { get; set; } = "continuous-tests";
+
+        [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment.")]
+        public bool CheckPeerConnection { get; set; } = false;
+       
+        public List<string> Validate()
+        {
+            var errors = new List<string>();
+
+            ForEachProperty(
+                onString: (n, v) => StringIsSet(n, v, errors),
+                onInt: (n, v) => IntIsOverZero(n, v, errors));
+
+            if (NumberOfValidators > NumberOfCodexNodes)
+            {
+                errors.Add($"{nameof(NumberOfValidators)} ({NumberOfValidators}) may not be greater than {nameof(NumberOfCodexNodes)} ({NumberOfCodexNodes}).");
+            }
+            if (StorageSell.HasValue && StorageQuota.HasValue && StorageSell.Value >= StorageQuota.Value)
+            {
+                errors.Add("StorageSell cannot be greater than or equal to StorageQuota.");
+            }
+
+            return errors;
+        }
+
+        private void ForEachProperty(Action<string, string> onString, Action<string, int?> onInt)
+        {
+            var properties = GetType().GetProperties();
+            foreach (var p in properties)
+            {
+                if (p.PropertyType == typeof(string)) onString(p.Name, (string)p.GetValue(this)!);
+                if (p.PropertyType == typeof(int?)) onInt(p.Name, (int?)p.GetValue(this)!);
+                if (p.PropertyType == typeof(int)) onInt(p.Name, (int)p.GetValue(this)!);
+            }
+        }
+
+        private static void IntIsOverZero(string variable, int? value, List<string> errors)
+        {
+            if (value == null || value.Value < 1)
+            {
+                errors.Add($"{variable} must be set and must be greater than 0.");
+            }
+        }
+
+        private static void StringIsSet(string variable, string value, List<string> errors)
+        {
+            if (string.IsNullOrWhiteSpace(value))
+            {
+                errors.Add($"{variable} must be set.");
+            }
+        }
+    }
+}

CodexNetDeployer/Deployer.cs

@@ -0,0 +1,152 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+using Logging;
+
+namespace CodexNetDeployer
+{
+    public class Deployer
+    {
+        private readonly Configuration config;
+        private readonly DefaultTimeSet timeset;
+        private readonly PeerConnectivityChecker peerConnectivityChecker;
+
+        public Deployer(Configuration config)
+        {
+            this.config = config;
+            timeset = new DefaultTimeSet();
+            peerConnectivityChecker = new PeerConnectivityChecker();
+        }
+
+        public CodexDeployment Deploy()
+        {
+            Log("Initializing...");
+            var lifecycle = CreateTestLifecycle();
+
+            Log("Preparing configuration...");
+            // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
+            var setup = new CodexSetup(999, config.CodexLogLevel);
+            setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
+            setup.MetricsMode = config.Metrics;
+
+            Log("Creating Geth instance and deploying contracts...");
+            var gethStarter = new GethStarter(lifecycle);
+            var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
+
+            Log("Geth started. Codex contracts deployed.");
+            Log("Warning: It can take up to 45 minutes for the Geth node to finish unlocking all if its 1000 preconfigured accounts.");
+
+            // It takes a second for the geth node to unlock a single account. Let's assume 3.
+            // We can't start the codex nodes until their accounts are definitely unlocked. So
+            // We wait:
+            Thread.Sleep(TimeSpan.FromSeconds(3.0 * config.NumberOfCodexNodes!.Value));
+
+            Log("Starting Codex nodes...");
+
+            // Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
+            var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
+            var startResults = new List<CodexNodeStartResult>();
+            for (var i = 0; i < config.NumberOfCodexNodes; i++)
+            {
+                var result = codexStarter.Start(i);
+                if (result != null) startResults.Add(result);
+            }
+
+            var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container));
+
+            CheckPeerConnectivity(startResults);
+            CheckContainerRestarts(startResults);
+
+            return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
+        }
+
+        private TestLifecycle CreateTestLifecycle()
+        {
+            var kubeConfig = GetKubeConfig(config.KubeConfigFile);
+
+            var lifecycleConfig = new DistTestCore.Configuration
+            (
+                kubeConfigFile: kubeConfig,
+                logPath: "null",
+                logDebug: false,
+                dataFilesPath: "notUsed",
+                codexLogLevel: config.CodexLogLevel,
+                k8sNamespacePrefix: config.KubeNamespace
+            );
+
+            return new TestLifecycle(new NullLog(), lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
+        }
+
+        private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable<RunningContainer> codexContainers)
+        {
+            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
+
+            Log("Starting metrics service...");
+            var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
+            var prometheusContainer = lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
+
+            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null);
+
+            Log("Starting dashboard service...");
+            var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer, setup);
+            return (prometheusContainer, grafanaStartInfo);
+        }
+
+        private string? GetKubeConfig(string kubeConfigFile)
+        {
+            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
+            return kubeConfigFile;
+        }
+
+        private void CheckPeerConnectivity(List<CodexNodeStartResult> codexContainers)
+        {
+            if (!config.CheckPeerConnection) return;
+
+            Log("Starting peer-connectivity check for deployed nodes...");
+            peerConnectivityChecker.CheckConnectivity(codexContainers);
+            Log("Check passed.");
+        }
+
+        private void CheckContainerRestarts(List<CodexNodeStartResult> startResults)
+        {
+            var crashes = new List<RunningContainer>();
+            foreach (var startResult in startResults)
+            {
+                var watcher = startResult.Workflow.CreateCrashWatcher(startResult.Container);
+                if (watcher.HasContainerCrashed()) crashes.Add(startResult.Container);
+            }
+
+            if (!crashes.Any())
+            {
+                Log("Container restart check passed.");
+            }
+            else
+            {
+                Log($"Deployment failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}");
+                throw new Exception("Deployment failed: One or more containers crashed.");
+            }
+        }
+
+        private DeploymentMetadata CreateMetadata()
+        {
+            return new DeploymentMetadata(
+                kubeNamespace: config.KubeNamespace,
+                numberOfCodexNodes: config.NumberOfCodexNodes!.Value,
+                numberOfValidators: config.NumberOfValidators!.Value,
+                storageQuotaMB: config.StorageQuota!.Value,
+                codexLogLevel: config.CodexLogLevel,
+                initialTestTokens: config.InitialTestTokens,
+                minPrice: config.MinPrice,
+                maxCollateral: config.MaxCollateral,
+                maxDuration: config.MaxDuration,
+                blockTTL: config.BlockTTL,
+                blockMI: config.BlockMI,
+                blockMN: config.BlockMN);
+        }
+
+        private void Log(string msg)
+        {
+            Console.WriteLine(msg);
+        }
+    }
+}

CodexNetDeployer/PeerConnectivityChecker.cs

@@ -0,0 +1,34 @@
+using DistTestCore.Helpers;
+using Logging;
+
+namespace CodexNetDeployer
+{
+    public class PeerConnectivityChecker
+    {
+        public void CheckConnectivity(List<CodexNodeStartResult> startResults)
+        {
+            var log = new ConsoleLog();
+            var checker = new PeerConnectionTestHelpers(log);
+            var access = startResults.Select(r => r.Access);
+
+            checker.AssertFullyConnected(access);
+        }
+    }
+
+    public class ConsoleLog : BaseLog
+    {
+        public ConsoleLog() : base(false)
+        {
+        }
+
+        protected override string GetFullName()
+        {
+            return "CONSOLE";
+        }
+
+        public override void Log(string message)
+        {
+            Console.WriteLine(message);
+        }
+    }
+}

CodexNetDeployer/Program.cs

@@ -0,0 +1,60 @@
+using ArgsUniform;
+using CodexNetDeployer;
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
+using Newtonsoft.Json;
+using Configuration = CodexNetDeployer.Configuration;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        var nl = Environment.NewLine;
+        Console.WriteLine("CodexNetDeployer" + nl);
+
+        var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
+        var config = uniformArgs.Parse(true);
+        
+        var errors = config.Validate();
+        if (errors.Any())
+        {
+            Console.WriteLine($"Configuration errors: ({errors.Count})");
+            foreach ( var error in errors ) Console.WriteLine("\t" + error);
+            Console.WriteLine(nl);
+            PrintHelp();
+            return;
+        }
+
+        Console.WriteLine("Using images:" + nl +
+            $"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl +
+            $"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl +
+            $"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl +
+            $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl +
+            $"\tGrafana image: '{new GrafanaContainerRecipe().Image}'" + nl);
+
+        if (!args.Any(a => a == "-y"))
+        {
+            Console.WriteLine("Does the above config look good? [y/n]");
+            if (Console.ReadLine()!.ToLowerInvariant() != "y") return;
+            Console.WriteLine("I think so too.");
+        }
+
+        var deployer = new Deployer(config);
+        var deployment = deployer.Deploy();
+
+        Console.WriteLine("Writing codex-deployment.json...");
+
+        File.WriteAllText("codex-deployment.json", JsonConvert.SerializeObject(deployment, Formatting.Indented));
+
+        Console.WriteLine("Done!");
+    }
+
+    private static void PrintHelp()
+    {
+        var nl = Environment.NewLine;
+        Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
+            "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
+            "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
+    }
+}

CodexNetDeployer/deploy-continuous-testnet.sh

@@ -0,0 +1,16 @@
+dotnet run \
+    --kube-config=/opt/kubeconfig.yaml \
+    --kube-namespace=codex-continuous-tests \
+    --nodes=5 \
+    --validators=3 \
+    --log-level=Trace \
+    --storage-quota=2048 \
+    --storage-sell=1024 \
+    --min-price=1024 \
+    --max-collateral=1024 \
+    --max-duration=3600000 \
+    --block-ttl=180 \
+    --block-mi=120 \
+    --block-mn=10000 \
+    --metrics=Dashboard \
+    --check-connect=1

CodexNetDownloader/CodexNetDownloader.csproj

@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
+    <ProjectReference Include="..\ContinuousTests\ContinuousTests.csproj" />
+    <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+  </ItemGroup>
+
+</Project>

CodexNetDownloader/Configuration.cs

@@ -0,0 +1,19 @@
+using ArgsUniform;
+using DistTestCore.Codex;
+
+namespace CodexNetDownloader
+{
+    public class Configuration
+    {
+        [Uniform("output-path", "o", "OUTPUT", true, "Path where files will be written.")]
+        public string OutputPath { get; set; } = "output";
+
+        [Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")]
+        public string CodexDeploymentJson { get; set; } = string.Empty;
+
+        [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
+        public string KubeConfigFile { get; set; } = "null";
+
+        public CodexDeployment CodexDeployment { get; set; } = null!;
+    }
+}

CodexNetDownloader/Program.cs

@@ -0,0 +1,48 @@
+using ArgsUniform;
+using ContinuousTests;
+using DistTestCore;
+using DistTestCore.Codex;
+using Logging;
+using Newtonsoft.Json;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        var nl = Environment.NewLine;
+        Console.WriteLine("CodexNetDownloader" + nl);
+
+        var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(PrintHelp, args);
+        var config = uniformArgs.Parse(true);
+
+        config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
+
+        if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
+
+        var k8sFactory = new K8sFactory();
+        var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
+
+        foreach (var container in config.CodexDeployment.CodexContainers)
+        {
+            lifecycle.DownloadLog(container);
+        }
+
+        Console.WriteLine("Done!");
+    }
+
+    private static CodexDeployment ParseCodexDeploymentJson(string filename)
+    {
+        var d = JsonConvert.DeserializeObject<CodexDeployment>(File.ReadAllText(filename))!;
+        if (d == null) throw new Exception("Unable to parse " + filename);
+        return d;
+    }
+
+    private static void PrintHelp()
+    {
+        var nl = Environment.NewLine;
+        Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
+
+        Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
+            "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
+    }
+}

ContinuousTests/CodexAccessFactory.cs

@@ -0,0 +1,20 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class CodexAccessFactory
+    {
+        public CodexAccess[] Create(Configuration config, RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
+        {
+            return containers.Select(container =>
+            {
+                var address = container.ClusterExternalAddress;
+                if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
+                return new CodexAccess(log, container, timeSet, address);
+            }).ToArray();
+        }
+    }
+}

ContinuousTests/Configuration.cs

@@ -0,0 +1,67 @@
+using ArgsUniform;
+using DistTestCore;
+using DistTestCore.Codex;
+using Newtonsoft.Json;
+
+namespace ContinuousTests
+{
+    public class Configuration
+    {
+        [Uniform("log-path", "l", "LOGPATH", true, "Path where log files will be written.")]
+        public string LogPath { get; set; } = "logs";
+
+        [Uniform("data-path", "d", "DATAPATH", true, "Path where temporary data files will be written.")]
+        public string DataPath { get; set; } = "data";
+
+        [Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")]
+        public string CodexDeploymentJson { get; set; } = string.Empty;
+
+        [Uniform("keep", "k", "KEEP", false, "Set to '1' to retain logs of successful tests.")]
+        public bool KeepPassedTestLogs { get; set; } = false;
+
+        [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
+        public string KubeConfigFile { get; set; } = "null";
+
+        [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
+        public bool StopOnFailure { get; set; } = false;
+
+        [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
+        public bool DownloadContainerLogs { get; set; } = false;
+
+        public CodexDeployment CodexDeployment { get; set; } = null!;
+
+        public RunnerLocation RunnerLocation { get; set; }
+    }
+
+    public class ConfigLoader
+    {
+        public Configuration Load(string[] args)
+        {
+            var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
+
+            var result = uniformArgs.Parse(true);
+            
+            result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
+            result.RunnerLocation = RunnerLocationUtils.DetermineRunnerLocation(result.CodexDeployment.CodexContainers.First());
+
+            return result;
+        }
+        
+        private CodexDeployment ParseCodexDeploymentJson(string filename)
+        {
+            var d = JsonConvert.DeserializeObject<CodexDeployment>(File.ReadAllText(filename))!;
+            if (d == null) throw new Exception("Unable to parse " + filename);
+            return d;
+        }
+
+        private static void PrintHelp()
+        {
+            var nl = Environment.NewLine;
+            Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
+                "The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
+
+            Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
+                "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
+        }
+    }
+}

ContinuousTests/ContinuousTest.cs

@@ -0,0 +1,119 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using DistTestCore.Logs;
+using KubernetesWorkflow;
+using Logging;
+
+namespace ContinuousTests
+{
+    public abstract class ContinuousTestLongTimeouts : ContinuousTest
+    {
+        public override ITimeSet TimeSet => new LongTimeSet();
+    }
+
+    public abstract class ContinuousTest
+    {
+        protected const int Zero = 0;
+        protected const int MinuteOne = 60;
+        protected const int MinuteFive = MinuteOne * 5;
+        protected const int HourOne = MinuteOne * 60;
+        protected const int HourThree = HourOne * 3;
+        protected const int DayOne = HourOne * 24;
+        protected const int DayThree = DayOne * 3;
+
+        private const string UploadFailedMessage = "Unable to store block";
+
+        public void Initialize(CodexAccess[] nodes, BaseLog log, FileManager fileManager, Configuration configuration, CancellationToken cancelToken)
+        {
+            Nodes = nodes;
+            Log = log;
+            FileManager = fileManager;
+            Configuration = configuration;
+            CancelToken = cancelToken;
+
+            if (nodes != null)
+            {
+                NodeRunner = new NodeRunner(Nodes, configuration, TimeSet, Log, CustomK8sNamespace, EthereumAccountIndex);
+            }
+            else
+            {
+                NodeRunner = null!;
+            }
+        }
+
+        public CodexAccess[] Nodes { get; private set; } = null!;
+        public BaseLog Log { get; private set; } = null!;
+        public IFileManager FileManager { get; private set; } = null!;
+        public Configuration Configuration { get; private set; } = null!;
+        public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
+        public CancellationToken CancelToken { get; private set; } = new CancellationToken();
+        public NodeRunner NodeRunner { get; private set; } = null!;
+
+        public abstract int RequiredNumberOfNodes { get; }
+        public abstract TimeSpan RunTestEvery { get; }
+        public abstract TestFailMode TestFailMode { get; }
+        public virtual int EthereumAccountIndex { get { return -1; } }
+        public virtual string CustomK8sNamespace { get { return string.Empty; } }
+
+        public string Name
+        {
+            get
+            {
+                return GetType().Name;
+            }
+        }
+
+        public ContentId? UploadFile(CodexAccess node, TestFile file)
+        {
+            using var fileStream = File.OpenRead(file.Filename);
+
+            var logMessage = $"Uploading file {file.Describe()}...";
+            var response = Stopwatch.Measure(Log, logMessage, () =>
+            {
+                return node.UploadFile(fileStream);
+            });
+
+            if (string.IsNullOrEmpty(response)) return null;
+            if (response.StartsWith(UploadFailedMessage)) return null;
+            
+            Log.Log($"Uploaded file. Received contentId: '{response}'.");
+            return new ContentId(response);
+        }
+
+        public TestFile DownloadFile(CodexAccess node, ContentId contentId, string fileLabel = "")
+        {
+            var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
+            var file = FileManager.CreateEmptyTestFile(fileLabel);
+            Stopwatch.Measure(Log, logMessage, () => DownloadToFile(node, contentId.Id, file));
+            Log.Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
+            return file;
+        }
+
+        public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null)
+        {
+            var nodeRunner = new NodeRunner(Nodes, Configuration, TimeSet, Log, Configuration.CodexDeployment.Metadata.KubeNamespace, EthereumAccountIndex);
+            return nodeRunner.DownloadLog(container, tailLines);
+        }
+
+        private void DownloadToFile(CodexAccess node, string contentId, TestFile file)
+        {
+            using var fileStream = File.OpenWrite(file.Filename);
+            try
+            {
+                using var downloadStream = node.DownloadFile(contentId);
+                downloadStream.CopyTo(fileStream);
+            }
+            catch
+            {
+                Log.Log($"Failed to download file '{contentId}'.");
+                throw;
+            }
+        }
+    }
+
+    public enum TestFailMode
+    {
+        StopAfterFirstFailure,
+        AlwaysRunAllMoments
+    }
+}

ContinuousTests/ContinuousTestRunner.cs

@@ -0,0 +1,65 @@
+using DistTestCore;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class ContinuousTestRunner
+    {
+        private readonly K8sFactory k8SFactory = new K8sFactory();
+        private readonly ConfigLoader configLoader = new ConfigLoader();
+        private readonly TestFactory testFactory = new TestFactory();
+        private readonly Configuration config;
+        private readonly StartupChecker startupChecker;
+        private readonly CancellationToken cancelToken;
+
+        public ContinuousTestRunner(string[] args, CancellationToken cancelToken)
+        {
+            config = configLoader.Load(args);
+            startupChecker = new StartupChecker(config, cancelToken);
+            this.cancelToken = cancelToken;
+        }
+
+        public void Run()
+        {
+            startupChecker.Check();
+
+            var taskFactory = new TaskFactory();
+            var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "Overview");
+            overviewLog.Log("Continuous tests starting...");
+            var allTests = testFactory.CreateTests();
+
+            ClearAllCustomNamespaces(allTests, overviewLog);
+
+            var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
+
+            foreach (var testLoop in testLoops)
+            {
+                if (cancelToken.IsCancellationRequested) break;
+
+                overviewLog.Log("Launching test-loop for " + testLoop.Name);
+                testLoop.Begin();
+                Thread.Sleep(TimeSpan.FromSeconds(5));
+            }
+
+            overviewLog.Log("Finished launching test-loops.");
+            cancelToken.WaitHandle.WaitOne();
+            overviewLog.Log("Cancelling all test-loops...");
+            taskFactory.WaitAll();
+            overviewLog.Log("All tasks cancelled.");
+        }
+
+        private void ClearAllCustomNamespaces(ContinuousTest[] allTests, FixtureLog log)
+        {
+            foreach (var test in allTests) ClearAllCustomNamespaces(test, log);
+        }
+
+        private void ClearAllCustomNamespaces(ContinuousTest test, FixtureLog log)
+        {
+            if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
+
+            log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
+            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log);
+            lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources();
+        }
+    }
+}

ContinuousTests/ContinuousTests.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net7.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ArgsUniform\ArgsUniform.csproj" />
+    <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
+    <ProjectReference Include="..\KubernetesWorkflow\KubernetesWorkflow.csproj" />
+    <ProjectReference Include="..\Logging\Logging.csproj" />
+  </ItemGroup>
+
+</Project>

ContinuousTests/K8sFactory.cs

@@ -0,0 +1,31 @@
+using DistTestCore.Codex;
+using DistTestCore;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class K8sFactory
+    {
+        public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log)
+        {
+            var kubeConfig = GetKubeConfig(kubeConfigFile);
+            var lifecycleConfig = new DistTestCore.Configuration
+            (
+                kubeConfigFile: kubeConfig,
+                logPath: logPath,
+                logDebug: false,
+                dataFilesPath: dataFilePath,
+                codexLogLevel: CodexLogLevel.Debug,
+                k8sNamespacePrefix: customNamespace
+            );
+
+            return new TestLifecycle(log, lifecycleConfig, timeSet, "continuous-tests", string.Empty);
+        }
+
+        private static string? GetKubeConfig(string kubeConfigFile)
+        {
+            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
+            return kubeConfigFile;
+        }
+    }
+}

ContinuousTests/NodeRunner.cs

@@ -0,0 +1,113 @@
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using DistTestCore;
+using KubernetesWorkflow;
+using NUnit.Framework;
+using Logging;
+using Utils;
+using DistTestCore.Logs;
+
+namespace ContinuousTests
+{
+    public class NodeRunner
+    {
+        private readonly K8sFactory k8SFactory = new K8sFactory();
+        private readonly CodexAccess[] nodes;
+        private readonly Configuration config;
+        private readonly ITimeSet timeSet;
+        private readonly BaseLog log;
+        private readonly string customNamespace;
+        private readonly int ethereumAccountIndex;
+
+        public NodeRunner(CodexAccess[] nodes, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex)
+        {
+            this.nodes = nodes;
+            this.config = config;
+            this.timeSet = timeSet;
+            this.log = log;
+            this.customNamespace = customNamespace;
+            this.ethereumAccountIndex = ethereumAccountIndex;
+        }
+
+        public void RunNode(Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
+        {
+            RunNode(nodes.ToList().PickOneRandom(), operation, 0.TestTokens());
+        }
+
+        public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation)
+        {
+            RunNode(bootstrapNode, operation, 0.TestTokens());
+        }
+
+        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
+        {
+            var subFile = log.CreateSubfile();
+            var description = container.Name;
+            var handler = new LogDownloadHandler(container, description, subFile);
+
+            log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
+
+            var lifecycle = CreateTestLifecycle();
+            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
+            flow.DownloadContainerLog(container, handler, tailLines);
+
+            return new DownloadedLog(subFile, description);
+        }
+
+        public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
+        {
+            var lifecycle = CreateTestLifecycle();
+            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
+
+            try
+            {
+                var debugInfo = bootstrapNode.GetDebugInfo();
+                Assert.That(!string.IsNullOrEmpty(debugInfo.spr));
+
+                var startupConfig = new StartupConfig();
+                startupConfig.NameOverride = "TransientNode";
+                var codexStartConfig = new CodexStartupConfig(CodexLogLevel.Trace);
+                codexStartConfig.MarketplaceConfig = new MarketplaceInitialConfig(0.Eth(), 0.TestTokens(), false);
+                codexStartConfig.MarketplaceConfig.AccountIndexOverride = ethereumAccountIndex;
+                codexStartConfig.BootstrapSpr = debugInfo.spr;
+                startupConfig.Add(codexStartConfig);
+                startupConfig.Add(config.CodexDeployment.GethStartResult);
+                var rc = flow.Start(1, Location.Unspecified, new CodexContainerRecipe(), startupConfig);
+
+                var account = config.CodexDeployment.GethStartResult.CompanionNode.Accounts[ethereumAccountIndex];
+
+                var marketplaceNetwork = config.CodexDeployment.GethStartResult.MarketplaceNetwork;
+                if (mintTestTokens.Amount > 0)
+                {
+                    var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
+                    var interaction = marketplaceNetwork.Bootstrap.StartInteraction(lifecycle);
+                    interaction.MintTestTokens(new[] { account.Account }, mintTestTokens.Amount, tokenAddress);
+                }
+
+                var container = rc.Containers[0];
+                var address = lifecycle.Configuration.GetAddress(container);
+                var codexAccess = new CodexAccess(log, container, lifecycle.TimeSet, address);
+                var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess);
+
+                try
+                {
+                    operation(codexAccess, marketAccess, lifecycle);
+                }
+                catch
+                {
+                    lifecycle.DownloadLog(container);
+                    throw;
+                }
+            }
+            finally
+            {
+                flow.DeleteTestResources();
+            }
+        }
+
+        private TestLifecycle CreateTestLifecycle()
+        {
+            return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log);
+        }
+    }
+}

ContinuousTests/Program.cs

@@ -0,0 +1,33 @@
+using ContinuousTests;
+
+public class Program
+{
+    public static void Main(string[] args)
+    {
+        Console.WriteLine("Codex Continous-Test-Runner.");
+        Console.WriteLine("Running...");
+
+        var runner = new ContinuousTestRunner(args, Cancellation.Cts.Token);
+
+        Console.CancelKeyPress += (sender, e) =>
+        {
+            Console.WriteLine("Stopping...");
+            e.Cancel = true;
+
+            Cancellation.Cts.Cancel();
+        };
+        
+        runner.Run();
+        Console.WriteLine("Done.");
+    }
+
+    public static class Cancellation
+    {
+        static Cancellation()
+        {
+            Cts = new CancellationTokenSource();
+        }
+
+        public static CancellationTokenSource Cts { get; } 
+    }
+}

ContinuousTests/SingleTestRun.cs

@@ -0,0 +1,227 @@
+using DistTestCore.Codex;
+using DistTestCore;
+using Logging;
+using Utils;
+using KubernetesWorkflow;
+using NUnit.Framework.Internal;
+using System.Reflection;
+using static Program;
+
+namespace ContinuousTests
+{
+    public class SingleTestRun
+    {
+        private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
+        private readonly List<Exception> exceptions = new List<Exception>();
+        private readonly TaskFactory taskFactory;
+        private readonly Configuration config;
+        private readonly BaseLog overviewLog;
+        private readonly TestHandle handle;
+        private readonly CancellationToken cancelToken;
+        private readonly CodexAccess[] nodes;
+        private readonly FileManager fileManager;
+        private readonly FixtureLog fixtureLog;
+        private readonly string testName;
+        private readonly string dataFolder;
+
+        public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, CancellationToken cancelToken)
+        {
+            this.taskFactory = taskFactory;
+            this.config = config;
+            this.overviewLog = overviewLog;
+            this.handle = handle;
+            this.cancelToken = cancelToken;
+            testName = handle.Test.GetType().Name;
+            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
+
+            nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
+            dataFolder = config.DataPath + "-" + Guid.NewGuid();
+            fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
+        }
+
+        public void Run(EventWaitHandle runFinishedHandle)
+        {
+            taskFactory.Run(() =>
+            {
+                try
+                {
+                    RunTest();
+                    fileManager.DeleteAllTestFiles();
+                    Directory.Delete(dataFolder, true);
+                    runFinishedHandle.Set();
+                }
+                catch (Exception ex)
+                {
+                    overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex);
+                    Environment.Exit(-1);
+                }
+            });
+        }
+
+        private void RunTest()
+        {
+            try
+            {
+                RunTestMoments();
+
+                if (!config.KeepPassedTestLogs) fixtureLog.Delete();
+            }
+            catch (Exception ex)
+            {
+                fixtureLog.Error("Test run failed with exception: " + ex);
+                fixtureLog.MarkAsFailed();
+
+                if (config.StopOnFailure)
+                {
+                    OverviewLog("Configured to stop on first failure. Downloading cluster logs...");
+                    DownloadClusterLogs();
+                    OverviewLog("Log download finished. Cancelling test runner...");
+                    Cancellation.Cts.Cancel();
+                }
+            }
+        }
+
+        private void RunTestMoments()
+        {
+            var earliestMoment = handle.GetEarliestMoment();
+
+            var t = earliestMoment;
+            while (true)
+            {
+                cancelToken.ThrowIfCancellationRequested();
+
+                RunMoment(t);
+
+                if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
+                {
+                    Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping...");
+                    ThrowFailTest();
+                }
+
+                var nextMoment = handle.GetNextMoment(t);
+                if (nextMoment != null)
+                {
+                    var delta = TimeSpan.FromSeconds(nextMoment.Value - t);
+                    Log($" > Next TestMoment in {Time.FormatDuration(delta)} seconds...");
+                    cancelToken.WaitHandle.WaitOne(delta);
+                    t = nextMoment.Value;
+                }
+                else
+                {
+                    if (exceptions.Any())
+                    {
+                        ThrowFailTest();
+                    }
+                    OverviewLog(" > Test passed. " + FuturesInfo());
+                    return;
+                }
+            }
+        }
+
+        private void ThrowFailTest()
+        {
+            var ex = UnpackException(exceptions.First());
+            Log(ex.ToString());
+            OverviewLog($" > Test failed {FuturesInfo()}: " + ex.Message);
+            throw ex;
+        }
+
+        private string FuturesInfo()
+        {
+            var containers = config.CodexDeployment.CodexContainers;
+            var nodes = codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
+            var f = nodes.Select(n => n.GetDebugFutures().ToString());
+            var msg = $"(Futures: [{string.Join(", ", f)}])";
+            return msg;
+        }
+
+        private void DownloadClusterLogs()
+        {
+            var k8sFactory = new K8sFactory();
+            var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
+
+            foreach (var container in config.CodexDeployment.CodexContainers)
+            {
+                lifecycle.DownloadLog(container);
+            }
+        }
+
+        private Exception UnpackException(Exception exception)
+        {
+            if (exception is AggregateException a)
+            {
+                return UnpackException(a.InnerExceptions.First());
+            }
+            if (exception is TargetInvocationException t)
+            {
+                return UnpackException(t.InnerException!);
+            }
+
+            return exception;
+        }
+
+        private void RunMoment(int t)
+        {
+            using (var context = new TestExecutionContext.IsolatedContext())
+            {
+                try
+                {
+                    handle.InvokeMoment(t, InitializeTest);
+                }
+                catch (Exception ex)
+                {
+                    exceptions.Add(ex);
+                }
+            }
+
+            DecommissionTest();
+        }
+
+        private void InitializeTest(string name)
+        {
+            Log($" > Running TestMoment '{name}'");
+            handle.Test.Initialize(nodes, fixtureLog, fileManager, config, cancelToken);
+        }
+
+        private void DecommissionTest()
+        {
+            handle.Test.Initialize(null!, null!, null!, null!, cancelToken);
+        }
+
+        private void Log(string msg)
+        {
+            fixtureLog.Log(msg);
+        }
+
+        private void OverviewLog(string msg)
+        {
+            Log(msg);
+            var containerNames = $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
+            overviewLog.Log($"{containerNames} {testName}: {msg}");
+        }
+
+        private CodexAccess[] CreateRandomNodes(int number)
+        {
+            var containers = SelectRandomContainers(number);
+            fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
+            return codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
+        }
+
+        private RunningContainer[] SelectRandomContainers(int number)
+        {
+            var containers = config.CodexDeployment.CodexContainers.ToList();
+            var result = new RunningContainer[number];
+            for (var i = 0; i < number; i++)
+            {
+                result[i] = containers.PickOneRandom();
+            }
+            return result;
+        }
+
+        private DistTestCore.Configuration CreateFileManagerConfiguration()
+        {
+            return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
+                CodexLogLevel.Error, string.Empty);
+        }
+    }
+}

ContinuousTests/StartupChecker.cs

@@ -0,0 +1,166 @@
+using DistTestCore.Codex;
+using DistTestCore;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class StartupChecker
+    {
+        private readonly TestFactory testFactory = new TestFactory();
+        private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
+        private readonly Configuration config;
+        private readonly CancellationToken cancelToken;
+
+        public StartupChecker(Configuration config, CancellationToken cancelToken)
+        {
+            this.config = config;
+            this.cancelToken = cancelToken;
+        }
+
+        public void Check()
+        {
+            var log = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "StartupChecks");
+            log.Log("Starting continuous test run...");
+            log.Log("Checking configuration...");
+            PreflightCheck(config);
+            log.Log("Contacting Codex nodes...");
+            CheckCodexNodes(log, config);
+            log.Log("All OK.");
+        }
+
+        private void PreflightCheck(Configuration config)
+        {
+            var tests = testFactory.CreateTests();
+            if (!tests.Any())
+            {
+                throw new Exception("Unable to find any tests.");
+            }
+            foreach (var test in tests)
+            {
+                cancelToken.ThrowIfCancellationRequested();
+
+                var handle = new TestHandle(test);
+                handle.GetEarliestMoment();
+                handle.GetLastMoment();
+            }
+
+            if (!Directory.Exists(config.LogPath))
+            {
+                Directory.CreateDirectory(config.LogPath);
+            }
+
+            var errors = CheckTests(tests);
+            if (errors.Any())
+            {
+                throw new Exception("Prerun check failed: " + string.Join(", ", errors));
+            }
+        }
+
+        private void CheckCodexNodes(BaseLog log, Configuration config)
+        {
+            var nodes = codexNodeFactory.Create(config, config.CodexDeployment.CodexContainers, log, new DefaultTimeSet());
+            var pass = true;
+            foreach (var n in nodes)
+            {
+                cancelToken.ThrowIfCancellationRequested();
+
+                log.Log($"Checking {n.Container.Name} @ '{n.Address.Host}:{n.Address.Port}'...");
+
+                if (EnsureOnline(log, n))
+                {
+                    log.Log("OK");
+                }
+                else
+                {
+                    log.Error($"No response from '{n.Address.Host}'.");
+                    pass = false;
+                }
+            }
+            if (!pass)
+            {
+                throw new Exception("Not all codex nodes responded.");
+            }
+        }
+
+        private bool EnsureOnline(BaseLog log, CodexAccess n)
+        {
+            try
+            {
+                var info = n.GetDebugInfo();
+                if (info == null || string.IsNullOrEmpty(info.id)) return false;
+
+                log.Log($"Codex version: '{info.codex.version}' revision: '{info.codex.revision}'");
+            }
+            catch
+            {
+                return false;
+            }
+            return true;
+        }
+
+        private List<string> CheckTests(ContinuousTest[] tests)
+        {
+            var errors = new List<string>();
+            CheckRequiredNumberOfNodes(tests, errors);
+            CheckCustomNamespaceClashes(tests, errors);
+            CheckEthereumIndexClashes(tests, errors);
+            return errors;
+        }
+
+        private void CheckEthereumIndexClashes(ContinuousTest[] tests, List<string> errors)
+        {
+            var offLimits = config.CodexDeployment.CodexContainers.Length;
+            foreach (var test in tests)
+            {
+                if (test.EthereumAccountIndex != -1)
+                {
+                    if (test.EthereumAccountIndex <= offLimits)
+                    {
+                        errors.Add($"Test '{test.Name}' has selected 'EthereumAccountIndex' = {test.EthereumAccountIndex}. All accounts up to and including {offLimits} are being used by the targetted Codex net. Select a different 'EthereumAccountIndex'.");
+                    }
+                }
+            }
+
+            DuplicatesCheck(tests, errors,
+                considerCondition: t => t.EthereumAccountIndex != -1,
+                getValue: t => t.EthereumAccountIndex,
+                propertyName: nameof(ContinuousTest.EthereumAccountIndex));
+        }
+
+        private void CheckCustomNamespaceClashes(ContinuousTest[] tests, List<string> errors)
+        {
+            DuplicatesCheck(tests, errors,
+                considerCondition: t => !string.IsNullOrEmpty(t.CustomK8sNamespace),
+                getValue: t => t.CustomK8sNamespace,
+                propertyName: nameof(ContinuousTest.CustomK8sNamespace));
+        }
+
+        private void DuplicatesCheck(ContinuousTest[] tests, List<string> errors, Func<ContinuousTest, bool> considerCondition, Func<ContinuousTest, object> getValue, string propertyName)
+        {
+            foreach (var test in tests)
+            {
+                if (considerCondition(test))
+                {
+                    var duplicates = tests.Where(t => t != test && getValue(t) == getValue(test)).ToList();
+                    if (duplicates.Any())
+                    {
+                        duplicates.Add(test);
+                        errors.Add($"Tests '{string.Join(",", duplicates.Select(d => d.Name))}' have the same '{propertyName}'. These must be unique.");
+                        return;
+                    }
+                }
+            }
+        }
+
+        private void CheckRequiredNumberOfNodes(ContinuousTest[] tests, List<string> errors)
+        {
+            foreach (var test in tests)
+            {
+                if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
+                {
+                    errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
+                }
+            }
+        }
+    }
+}

ContinuousTests/TaskFactory.cs

@@ -0,0 +1,39 @@
+namespace ContinuousTests
+{
+    public class TaskFactory
+    {
+        private readonly object taskLock = new();
+        private readonly List<Task> activeTasks = new List<Task>();
+
+        public void Run(Action action)
+        {
+            lock (taskLock)
+            {
+                activeTasks.Add(Task.Run(action).ContinueWith(CleanupTask, null));
+            }
+        }
+
+        public void WaitAll()
+        {
+            var tasks = activeTasks.ToArray();
+            Task.WaitAll(tasks);
+
+            var moreTasks = false;
+            lock (taskLock)
+            {
+                activeTasks.RemoveAll(task => task.IsCompleted);
+                moreTasks = activeTasks.Any();
+            }
+
+            if (moreTasks) WaitAll();
+        }
+
+        private void CleanupTask(Task completedTask, object? arg)
+        {
+            lock (taskLock)
+            {
+                activeTasks.Remove(completedTask);
+            }
+        }
+    }
+}

ContinuousTests/TestFactory.cs

@@ -0,0 +1,12 @@
+namespace ContinuousTests
+{
+    public class TestFactory
+    {
+        public ContinuousTest[] CreateTests()
+        {
+            var types = GetType().Assembly.GetTypes();
+            var testTypes = types.Where(t => typeof(ContinuousTest).IsAssignableFrom(t) && !t.IsAbstract);
+            return testTypes.Select(t => (ContinuousTest)Activator.CreateInstance(t)!).ToArray();
+        }
+    }
+}

ContinuousTests/TestHandle.cs

@@ -0,0 +1,86 @@
+using System.Reflection;
+
+namespace ContinuousTests
+{
+    public class TestHandle
+    {
+        private readonly List<MethodMoment> moments = new List<MethodMoment>();
+
+        public TestHandle(ContinuousTest test)
+        {
+            Test = test;
+
+            ReflectTestMoments();
+
+            var testName = test.GetType().Name;
+            if (!moments.Any()) throw new Exception($"Test '{testName}' has no moments.");
+            if (moments.Count != moments.Select(m => m.Moment).Distinct().Count()) throw new Exception($"Test '{testName}' has duplicate moments");
+        }
+
+        public ContinuousTest Test { get; }
+
+        public int GetEarliestMoment()
+        {
+            return moments.Min(m => m.Moment);
+        }
+
+        public int GetLastMoment()
+        {
+            return moments.Max(m => m.Moment);
+        }
+
+        public int? GetNextMoment(int currentMoment)
+        {
+            var remainingMoments = moments.Where(m => m.Moment > currentMoment).ToArray();
+            if (!remainingMoments.Any()) return null;
+            return remainingMoments.Min(m => m.Moment);
+        }
+
+        public void InvokeMoment(int currentMoment, Action<string> beforeInvoke)
+        {
+            var moment = moments.SingleOrDefault(m => m.Moment == currentMoment);
+            if (moment == null) return;
+
+            lock (MomentLock.Lock)
+            {
+                beforeInvoke(moment.Method.Name);
+                moment.Method.Invoke(Test, Array.Empty<object>());
+            }
+        }
+
+        private void ReflectTestMoments()
+        {
+            var methods = Test.GetType().GetMethods()
+                      .Where(m => m.GetCustomAttributes(typeof(TestMomentAttribute), false).Length > 0)
+                      .ToArray();
+
+            foreach (var method in methods)
+            {
+                var moment = method.GetCustomAttribute<TestMomentAttribute>();
+                if (moment != null && moment.T >= 0)
+                {
+                    moments.Add(new MethodMoment(method, moment.T));
+                }
+            }
+        }
+    }
+
+    public class MethodMoment
+    {
+        public MethodMoment(MethodInfo method, int moment)
+        {
+            Method = method;
+            Moment = moment;
+
+            if (moment < 0) throw new Exception("Moment must be zero or greater.");
+        }
+
+        public MethodInfo Method { get; }
+        public int Moment { get; }
+    }
+
+    public static class MomentLock
+    {
+        public static readonly object Lock = new();
+    }
+}

ContinuousTests/TestLoop.cs

@@ -0,0 +1,67 @@
+using Logging;
+
+namespace ContinuousTests
+{
+    public class TestLoop
+    {
+        private readonly TaskFactory taskFactory;
+        private readonly Configuration config;
+        private readonly BaseLog overviewLog;
+        private readonly Type testType;
+        private readonly TimeSpan runsEvery;
+        private readonly CancellationToken cancelToken;
+        private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset);
+
+        public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, CancellationToken cancelToken)
+        {
+            this.taskFactory = taskFactory;
+            this.config = config;
+            this.overviewLog = overviewLog;
+            this.testType = testType;
+            this.runsEvery = runsEvery;
+            this.cancelToken = cancelToken;
+            Name = testType.Name;
+        }
+
+        public string Name { get; }
+
+        public void Begin()
+        {
+            taskFactory.Run(() =>
+            {
+                try
+                {
+                    while (true)
+                    {
+                        WaitHandle.WaitAny(new[] { runFinishedHandle, cancelToken.WaitHandle });
+
+                        cancelToken.ThrowIfCancellationRequested();
+
+                        StartTest();
+
+                        cancelToken.WaitHandle.WaitOne(runsEvery);
+                    }
+                }
+                catch (OperationCanceledException)
+                {
+                    overviewLog.Log("Test-loop " + testType.Name + " is cancelled.");
+                }
+                catch (Exception ex)
+                {
+                    overviewLog.Error("Test infra failure: TestLoop failed with " + ex);
+                    Environment.Exit(-1);
+                }
+            });
+        }
+
+        private void StartTest()
+        {
+            var test = (ContinuousTest)Activator.CreateInstance(testType)!;
+            var handle = new TestHandle(test);
+            var run = new SingleTestRun(taskFactory, config, overviewLog, handle, cancelToken);
+
+            runFinishedHandle.Reset();
+            run.Run(runFinishedHandle);
+        }
+    }
+}

ContinuousTests/TestMomentAttribute.cs

@@ -0,0 +1,12 @@
+namespace ContinuousTests
+{
+    public class TestMomentAttribute : Attribute
+    {
+        public TestMomentAttribute(int t)
+        {
+            T = t;
+        }
+
+        public int T { get; }
+    }
+}

ContinuousTests/Tests/HoldMyBeerTest.cs

@@ -0,0 +1,39 @@
+using DistTestCore;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class HoldMyBeerTest : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => 1;
+        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(5);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+
+        private ContentId? cid;
+        private TestFile file = null!;
+
+        [TestMoment(t: Zero)]
+        public void UploadTestFile()
+        {
+            var metadata = Configuration.CodexDeployment.Metadata;
+            var maxQuotaUseMb = metadata.StorageQuotaMB / 2;
+            var safeTTL = Math.Max(metadata.BlockTTL, metadata.BlockMI) + 30;
+            var runsPerTtl = Convert.ToInt32(safeTTL / RunTestEvery.TotalSeconds);
+            var filesizePerUploadMb = Math.Min(80, maxQuotaUseMb / runsPerTtl);
+            // This filesize should keep the quota below 50% of the node's max.
+
+            var filesize = filesizePerUploadMb.MB();
+            double codexDefaultBlockSize = 31 * 64 * 33;
+            var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
+            Assert.That(numberOfBlocks, Is.EqualTo(1282));
+
+            file = FileManager.GenerateTestFile(filesize);
+
+            cid = UploadFile(Nodes[0], file);
+            Assert.That(cid, Is.Not.Null);
+
+            var dl = DownloadFile(Nodes[0], cid!);
+            file.AssertIsEqual(dl);
+        }
+    }
+}

ContinuousTests/Tests/MarketplaceTest.cs

@@ -0,0 +1,64 @@
+//using DistTestCore;
+//using DistTestCore.Codex;
+//using Newtonsoft.Json;
+//using NUnit.Framework;
+//using Utils;
+
+//namespace ContinuousTests.Tests
+//{
+//    public class MarketplaceTest : ContinuousTest
+//    {
+//        public override int RequiredNumberOfNodes => 1;
+//        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10);
+//        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+//        public override int EthereumAccountIndex => 200;
+//        public override string CustomK8sNamespace => "codex-continuous-marketplace";
+
+//        private readonly uint numberOfSlots = 3;
+//        private readonly ByteSize fileSize = 10.MB();
+//        private readonly TestToken pricePerSlotPerSecond = 10.TestTokens();
+
+//        private TestFile file = null!;
+//        private ContentId? cid;
+//        private string purchaseId = string.Empty;
+
+//        [TestMoment(t: Zero)]
+//        public void NodePostsStorageRequest()
+//        {
+//            var contractDuration = TimeSpan.FromMinutes(8);
+//            decimal totalDurationSeconds = Convert.ToDecimal(contractDuration.TotalSeconds);
+//            var expectedTotalCost = numberOfSlots * pricePerSlotPerSecond.Amount * (totalDurationSeconds + 1) * 1000000;
+
+//            file = FileManager.GenerateTestFile(fileSize);
+
+//            NodeRunner.RunNode((codexAccess, marketplaceAccess) =>
+//            {
+//                cid = UploadFile(codexAccess.Node, file);
+//                Assert.That(cid, Is.Not.Null);
+
+//                purchaseId = marketplaceAccess.RequestStorage(
+//                    contentId: cid!,
+//                    pricePerSlotPerSecond: pricePerSlotPerSecond,
+//                    requiredCollateral: 100.TestTokens(),
+//                    minRequiredNumberOfNodes: numberOfSlots,
+//                    proofProbability: 10,
+//                    duration: contractDuration);
+
+//                Assert.That(!string.IsNullOrEmpty(purchaseId));
+
+//                WaitForContractToStart(codexAccess, purchaseId);
+//            });
+//        }
+
+//        [TestMoment(t: MinuteFive + MinuteOne)]
+//        public void StoredDataIsAvailableAfterThreeDays()
+//        {
+//            NodeRunner.RunNode((codexAccess, marketplaceAccess) =>
+//            {
+//                var result = DownloadFile(codexAccess.Node, cid!);
+
+//                file.AssertIsEqual(result);
+//            });
+//        }
+//    }
+//}

ContinuousTests/Tests/PerformanceTests.cs

@@ -0,0 +1,86 @@
+//using DistTestCore;
+//using DistTestCore.Codex;
+//using NUnit.Framework;
+
+//namespace ContinuousTests.Tests
+//{
+//    public class UploadPerformanceTest : PerformanceTest
+//    {
+//        public override int RequiredNumberOfNodes => 1;
+
+//        [TestMoment(t: Zero)]
+//        public void UploadTest()
+//        {
+//            UploadTest(100, Nodes[0]);
+//        }
+//    }
+
+//    public class DownloadLocalPerformanceTest : PerformanceTest
+//    {
+//        public override int RequiredNumberOfNodes => 1;
+
+//        [TestMoment(t: Zero)]
+//        public void DownloadTest()
+//        {
+//            DownloadTest(100, Nodes[0], Nodes[0]);
+//        }
+//    }
+
+//    public class DownloadRemotePerformanceTest : PerformanceTest
+//    {
+//        public override int RequiredNumberOfNodes => 2;
+
+//        [TestMoment(t: Zero)]
+//        public void DownloadTest()
+//        {
+//            DownloadTest(100, Nodes[0], Nodes[1]);
+//        }
+//    }
+
+//    public abstract class PerformanceTest : ContinuousTest
+//    {
+//        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10);
+//        public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
+
+//        public void UploadTest(int megabytes, CodexAccess uploadNode)
+//        {
+//            var file = FileManager.GenerateTestFile(megabytes.MB());
+
+//            var time = Measure(() =>
+//            {
+//                UploadFile(uploadNode, file);
+//            });
+
+//            var timePerMB = time / megabytes;
+
+//            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached.");
+//        }
+
+//        public void DownloadTest(int megabytes, CodexAccess uploadNode, CodexAccess downloadNode)
+//        {
+//            var file = FileManager.GenerateTestFile(megabytes.MB());
+
+//            var cid = UploadFile(uploadNode, file);
+//            Assert.That(cid, Is.Not.Null);
+
+//            TestFile? result = null;
+//            var time = Measure(() =>
+//            {
+//                result = DownloadFile(downloadNode, cid!);
+//            });
+
+//            file.AssertIsEqual(result);
+
+//            var timePerMB = time / megabytes;
+
+//            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
+//        }
+
+//        private static TimeSpan Measure(Action action)
+//        {
+//            var start = DateTime.UtcNow;
+//            action();
+//            return DateTime.UtcNow - start;
+//        }
+//    }
+//}

ContinuousTests/Tests/ThresholdChecks.cs

@@ -0,0 +1,60 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class ThresholdChecks : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => 1;
+        public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+
+        private static readonly List<string> previousBreaches = new List<string>();
+
+        [TestMoment(t: 0)]
+        public void CheckAllThresholds()
+        {
+            var allNodes = CreateAccessToAllNodes();
+            foreach (var n in allNodes) CheckThresholds(n);
+        }
+
+        private void CheckThresholds(CodexAccess n)
+        {
+            var breaches = n.GetDebugThresholdBreaches();
+            if (breaches.breaches.Any())
+            {
+                var newBreaches = new List<string>();
+                foreach (var b in breaches.breaches)
+                {
+                    if (!previousBreaches.Contains(b))
+                    {
+                        newBreaches.Add(b);
+                        previousBreaches.Add(b);
+                    }
+                }
+
+                if (newBreaches.Any())
+                {
+                    Assert.Fail(string.Join(",", newBreaches.Select(b => FormatBreach(n, b))));
+
+                    Program.Cancellation.Cts.Cancel();
+                }
+            }
+        }
+
+        private string FormatBreach(CodexAccess n, string breach)
+        {
+            return $"{n.Container.Name} = '{breach}'";
+        }
+
+        private CodexAccess[] CreateAccessToAllNodes()
+        {
+            // Normally, a continuous test accesses only a subset of the nodes in the deployment.
+            // This time, we want to check all of them.
+            var factory = new CodexAccessFactory();
+            var allContainers = Configuration.CodexDeployment.CodexContainers;
+            return factory.Create(Configuration, allContainers, Log, new DefaultTimeSet());
+        }
+    }
+}

ContinuousTests/Tests/TransientNodeTest.cs

@@ -0,0 +1,54 @@
+//using DistTestCore;
+//using DistTestCore.Codex;
+//using NUnit.Framework;
+
+//namespace ContinuousTests.Tests
+//{
+//    public class TransientNodeTest : ContinuousTest
+//    {
+//        public override int RequiredNumberOfNodes => 3;
+//        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(1);
+//        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+//        public override string CustomK8sNamespace => nameof(TransientNodeTest).ToLowerInvariant();
+//        public override int EthereumAccountIndex => 201;
+
+//        private TestFile file = null!;
+//        private ContentId cid = null!;
+
+//        private CodexAccess UploadBootstapNode { get { return Nodes[0]; } }
+//        private CodexAccess DownloadBootstapNode { get { return Nodes[1]; } }
+//        private CodexAccess IntermediateNode { get { return Nodes[2]; } }
+
+//        [TestMoment(t: 0)]
+//        public void UploadWithTransientNode()
+//        {
+//            file = FileManager.GenerateTestFile(10.MB());
+
+//            NodeRunner.RunNode(UploadBootstapNode, (codexAccess, marketplaceAccess, lifecycle) =>
+//            {
+//                cid = UploadFile(codexAccess, file)!;
+//                Assert.That(cid, Is.Not.Null);
+
+//                var dlt = Task.Run(() =>
+//                {
+//                    Thread.Sleep(10000);
+//                    lifecycle.DownloadLog(codexAccess.Container);
+//                });
+
+//                var resultFile = DownloadFile(IntermediateNode, cid);
+//                dlt.Wait();
+//                file.AssertIsEqual(resultFile);
+//            });
+//        }
+
+//        [TestMoment(t: 30)]
+//        public void DownloadWithTransientNode()
+//        {
+//            NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess, lifecycle) =>
+//            {
+//                var resultFile = DownloadFile(codexAccess, cid);
+//                file.AssertIsEqual(resultFile);
+//            });
+//        }
+//    }
+//}

ContinuousTests/Tests/TwoClientTest.cs

@@ -0,0 +1,32 @@
+using DistTestCore;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class TwoClientTest : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => 2;
+        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(1);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+
+        private ContentId? cid;
+        private TestFile file = null!;
+
+        [TestMoment(t: Zero)]
+        public void UploadTestFile()
+        {
+            file = FileManager.GenerateTestFile(10.MB());
+
+            cid = UploadFile(Nodes[0], file);
+            Assert.That(cid, Is.Not.Null);
+        }
+
+        [TestMoment(t: 10)]
+        public void DownloadTestFile()
+        {
+            var dl = DownloadFile(Nodes[1], cid!);
+
+            file.AssertIsEqual(dl);
+        }
+    }
+}

ContinuousTests/reports/CodexTestNetReport-July2023.md

@@ -0,0 +1,38 @@
+# Codex Continuous Test-net Report
+Date: 02-08-2023
+
+Report for: 07-2023
+
+
+## Test-net Status
+- Start of month: Offline - faulted
+- End of month: Offline - faulted
+
+(Faulted: Tests fail with such frequency that the information gathered does not justify the cost of leaving the test-net running.)
+
+## Deployment Configuration
+Continous Test-net is deployed to the kubernetes cluster with the following configuration:
+
+5x Codex Nodes:
+- Log-level: Trace
+- Storage quota: 2048 MB
+- Storage sell: 1024 MB
+- Min price: 1024
+- Max collateral: 1024
+- Max duration: 3600000 seconds
+- Block-TTL: 120 seconds
+
+3 of these 5 nodes have:
+- Validator: true
+
+Kubernetes namespace: 'codex-continuous-tests'
+
+## Test Overview
+| Changes        | Test                | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | Status      | Results                                                                              |
+|----------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|--------------------------------------------------------------------------------------|
+| New in 07-2023 | Two-client test     | Every 30 seconds, two nodes are chosen at random. A 10 MB file is generated and uploaded to one. 10 seconds later, it is downloaded from the other. File contents are asserted to be equal.                                                                                                                                                                                                                                                                                                                                                                                    | Faulted     | Test reliably fails after 30 to 45 minutes. Both upload and download failures occur. |
+| New in 07-2023 | Transient-node test | Every 1 minute, a new, transient Codex node is started and bootstrapped against a random node of the test net. A 10 MB file is generated and uploaded to the transient node. The file is then downloaded from a second random test net node and file equality is asserted. After that, the transient node is shut down. 30 seconds later, a new transient Codex node is started and bootstrapped against a third (guaranteed different from first and second) node from the test net. The same file is then downloaded from the new transient node. File equality is asserted. | Not running | Test was not run because the previous more rudamentory test has faulted.             |
+
+## Action Points
+- Codex logs (from these long-running containers) are often incomplete when downloaded after a test failure. A reliable way of maintaining these logs is needed. The logs can quickly explode in size, even for test-nets that run only for a few hours.
+- The distributed testing setup can and has been used to reproduce the failure of the Two-client test in a local environment. Investigation is on-going.

ContinuousTests/run.sh

@@ -0,0 +1,6 @@
+dotnet run \
+    --kube-config=/opt/kubeconfig.yaml \
+    --codex-deployment=codex-deployment.json \
+    --keep=1 \
+    --stop=1 \
+    --dl-logs=1

DistTestCore/AutoBootstrapDistTest.cs

@@ -0,0 +1,30 @@
+using NUnit.Framework;
+
+namespace DistTestCore
+{
+    public class AutoBootstrapDistTest : DistTest
+    {
+        public override IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
+        {
+            throw new Exception("AutoBootstrapDistTest creates and attaches a single bootstrap node for you. " +
+                "If you want to control the bootstrap node from your test, please use DistTest instead.");
+        }
+
+        public override ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        {
+            var codexSetup = CreateCodexSetup(numberOfNodes);
+            setup(codexSetup);
+            codexSetup.WithBootstrapNode(BootstrapNode);
+            return BringOnline(codexSetup);
+        }
+
+        [SetUp]
+        public void SetUpBootstrapNode()
+        {
+            var setup = CreateCodexSetup(1).WithName("BOOTSTRAP");
+            BootstrapNode = BringOnline(setup)[0];
+        }
+
+        protected IOnlineCodexNode BootstrapNode { get; private set; } = null!;
+    }
+}

DistTestCore/BaseStarter.cs

@@ -0,0 +1,42 @@
+using Logging;
+
+namespace DistTestCore
+{
+    public class BaseStarter
+    {
+        protected readonly TestLifecycle lifecycle;
+        private Stopwatch? stopwatch;
+
+        public BaseStarter(TestLifecycle lifecycle)
+        {
+            this.lifecycle = lifecycle;
+        }
+
+        protected void LogStart(string msg)
+        {
+            Log(msg);
+            stopwatch = Stopwatch.Begin(lifecycle.Log, GetClassName());
+        }
+
+        protected void LogEnd(string msg)
+        {
+            stopwatch!.End(msg);
+            stopwatch = null;
+        }
+
+        protected void Log(string msg)
+        {
+            lifecycle.Log.Log($"{GetClassName()} {msg}");
+        }
+
+        protected void Debug(string msg)
+        {
+            lifecycle.Log.Debug($"{GetClassName()} {msg}", 1);
+        }
+
+        private string GetClassName()
+        {
+            return $"({GetType().Name})";
+        }
+    }
+}

DistTestCore/ByteSize.cs

@@ -1,19 +1,48 @@
-namespace DistTestCore
+using Utils;
+
+namespace DistTestCore
 {
     public class ByteSize
     {
+
         public ByteSize(long sizeInBytes)
         {
+            if (sizeInBytes < 0) throw new ArgumentException("Cannot create ByteSize object with size less than 0. Was: " + sizeInBytes);
             SizeInBytes = sizeInBytes;
         }
 
         public long SizeInBytes { get; }
+
+        public long ToMB()
+        {
+            return SizeInBytes / (1024 * 1024);
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is ByteSize size && SizeInBytes == size.SizeInBytes;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(SizeInBytes);
+        }
+
+        public override string ToString()
+        {
+            return Formatter.FormatByteSize(SizeInBytes);
+        }
     }
 
-    public static class IntExtensions
+    public static class ByteSizeIntExtensions
     {
         private const long Kilo = 1024;
 
+        public static ByteSize Bytes(this long i)
+        {
+            return new ByteSize(i);
+        }
+
         public static ByteSize KB(this long i)
         {
             return new ByteSize(i * Kilo);
@@ -34,6 +63,11 @@
             return (i * Kilo).GB();
         }
 
+        public static ByteSize Bytes(this int i)
+        {
+            return new ByteSize(i);
+        }
+
         public static ByteSize KB(this int i)
         {
             return Convert.ToInt64(i).KB();

DistTestCore/Codex/CodexAccess.cs

@@ -1,21 +1,63 @@
 using KubernetesWorkflow;
+using Logging;
+using Utils;
 
 namespace DistTestCore.Codex
 {
-    public class CodexAccess
+    public class CodexAccess : ILogHandler
     {
-        public CodexAccess(RunningContainer runningContainer)
+        private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
+        private bool hasContainerCrashed;
+
+        public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
         {
-            Container = runningContainer;
+            this.log = log;
+            Container = container;
+            this.timeSet = timeSet;
+            Address = address;
+            hasContainerCrashed = false;
+
+            if (container.CrashWatcher != null) container.CrashWatcher.Start(this);
         }
 
         public RunningContainer Container { get; }
+        public Address Address { get; }
 
         public CodexDebugResponse GetDebugInfo()
         {
             return Http().HttpGetJson<CodexDebugResponse>("debug/info");
         }
 
+        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        {
+            var http = Http();
+            var str = http.HttpGetString($"debug/peer/{peerId}");
+
+            if (str.ToLowerInvariant() == "unable to find peer!")
+            {
+                return new CodexDebugPeerResponse
+                {
+                    IsPeerFound = false
+                };
+            }
+
+            var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
+            result.IsPeerFound = true;
+            return result;
+        }
+
+        public int GetDebugFutures()
+        {
+            // Some Codex images support debug/futures to count the number of open futures.
+            return 0; // Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
+        }
+
+        public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
+        {
+            return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");
+        }
+
         public string UploadFile(FileStream fileStream)
         {
             return Http().HttpPostStream("upload", fileStream);
@@ -26,31 +68,56 @@ namespace DistTestCore.Codex
             return Http().HttpGetStream("download/" + contentId);
         }
 
-        private Http Http()
+        public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
         {
-            var ip = Container.Pod.Cluster.IP;
-            var port = Container.ServicePorts[0].Number;
-            return new Http(ip, port, baseUrl: "/api/codex/v1");
+            return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
+        }
+
+        public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
+        {
+            return Http().HttpPostJson($"storage/request/{contentId}", request);
+        }
+
+        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
+        {
+            return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
         }
 
         public string ConnectToPeer(string peerId, string peerMultiAddress)
         {
             return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
         }
-    }
 
-    public class CodexDebugResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string[] addrs { get; set; } = new string[0];
-        public string repo { get; set; } = string.Empty;
-        public string spr { get; set; } = string.Empty;
-        public CodexDebugVersionResponse codex { get; set; } = new();
-    }
+        public string GetName()
+        {
+            return Container.Name;
+        }
 
-    public class CodexDebugVersionResponse
-    {
-        public string version { get; set; } = string.Empty;
-        public string revision { get; set; } = string.Empty;
+        private Http Http()
+        {
+            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
+        }
+
+        private void CheckContainerCrashed(HttpClient client)
+        {
+            if (hasContainerCrashed) throw new Exception("Container has crashed.");
+        }
+
+        public void Log(Stream crashLog)
+        {
+            var file = log.CreateSubfile();
+            log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
+
+            using var reader = new StreamReader(crashLog);
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                file.Write(line);
+                line = reader.ReadLine();
+            }
+
+            log.Log("Crash log successfully downloaded.");
+            hasContainerCrashed = true;
+        }
     }
 }

DistTestCore/Codex/CodexApiTypes.cs

@@ -0,0 +1,128 @@
+using KubernetesWorkflow;
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+
+namespace DistTestCore.Codex
+{
+    public class CodexDebugResponse
+    {
+        public string id { get; set; } = string.Empty;
+        public string[] addrs { get; set; } = new string[0];
+        public string repo { get; set; } = string.Empty;
+        public string spr { get; set; } = string.Empty;
+        public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
+        public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
+        public CodexDebugVersionResponse codex { get; set; } = new();
+        public CodexDebugTableResponse table { get; set; } = new();
+    }
+
+    public class CodexDebugFutures
+    {
+        public int futures { get; set; }
+    }
+
+    public class CodexDebugTableResponse
+    {
+        public CodexDebugTableNodeResponse localNode { get; set; } = new();
+        public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
+    }
+
+    public class CodexDebugTableNodeResponse
+    {
+        public string nodeId { get; set; } = string.Empty;
+        public string peerId { get; set; } = string.Empty;
+        public string record { get; set; } = string.Empty;
+        public string address { get; set; } = string.Empty;
+        public bool seen { get; set; }
+    }
+
+    public class EnginePeerResponse
+    {
+        public string peerId { get; set; } = string.Empty;
+        public EnginePeerContextResponse context { get; set; } = new();
+    }
+
+    public class EnginePeerContextResponse
+    {
+        public int blocks { get; set; } = 0;
+        public int peerWants { get; set; } = 0;
+        public int exchanged { get; set; } = 0;
+        public string lastExchange { get; set; } = string.Empty;
+    }
+
+    public class SwitchPeerResponse
+    {
+        public string peerId { get; set; } = string.Empty;
+        public string key { get; set; } = string.Empty;
+    }
+
+    public class CodexDebugVersionResponse
+    {
+        public string version { get; set; } = string.Empty;
+        public string revision { get; set; } = string.Empty;
+
+        public bool IsValid()
+        {
+            return !string.IsNullOrEmpty(version) && !string.IsNullOrEmpty(revision);
+        }
+
+        public override string ToString()
+        {
+            return JsonConvert.SerializeObject(this);
+        }
+    }
+
+    public class CodexDebugPeerResponse
+    {
+        public bool IsPeerFound { get; set; }
+
+        public string peerId { get; set; } = string.Empty;
+        public long seqNo { get; set; }
+        public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
+    }
+
+    public class CodexDebugPeerAddressResponse
+    {
+        public string address { get; set; } = string.Empty;
+    }
+
+    public class CodexDebugThresholdBreaches
+    {
+        public string[] breaches { get; set; } = Array.Empty<string>();
+    }
+
+    public class CodexSalesAvailabilityRequest
+    {
+        public string size { get; set; } = string.Empty;
+        public string duration { get; set; } = string.Empty;
+        public string minPrice { get; set; } = string.Empty;
+        public string maxCollateral { get; set; } = string.Empty;
+    }
+
+    public class CodexSalesAvailabilityResponse
+    {
+        public string id { get; set; } = string.Empty;
+        public string size { get; set; } = string.Empty;
+        public string duration { get; set; } = string.Empty;
+        public string minPrice { get; set; } = string.Empty;
+        public string maxCollateral { get; set; } = string.Empty;
+    }
+
+    public class CodexSalesRequestStorageRequest
+    {
+        public string duration { get; set; } = string.Empty;
+        public string proofProbability { get; set; } = string.Empty;
+        public string reward { get; set; } = string.Empty;
+        public string collateral { get; set; } = string.Empty;
+        public string? expiry { get; set; }
+        public uint? nodes { get; set; }
+        public uint? tolerance { get; set; }
+    }
+
+    public class CodexStoragePurchase
+    {
+        public string state { get; set; } = string.Empty;
+        public string error { get; set; } = string.Empty;
+    }
+}

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -5,50 +5,100 @@ namespace DistTestCore.Codex
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        public const string DockerImage = "thatbenbierens/nim-codex:sha-b204837";
-        public const string MetricsPortTag = "metrics_port";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
 
-        protected override string Image => DockerImage;
+        public const string MetricsPortTag = "metrics_port";
+        public const string DiscoveryPortTag = "discovery-port";
+
+        // Used by tests for time-constraint assersions.
+        public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
+        public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
+
+        public override string AppName => "codex";
+        public override string Image { get; }
+
+        public CodexContainerRecipe()
+        {
+            Image = GetDockerImage();
+        }
 
         protected override void Initialize(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<CodexStartupConfig>();
 
-            AddExposedPortAndVar("API_PORT");
-            AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}");
-            AddInternalPortAndVar("DISC_PORT");
+            AddExposedPortAndVar("CODEX_API_PORT");
+            AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
+
+            AddEnvVar("CODEX_DATA_DIR", $"datadir{ContainerNumber}");
+            AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
+            AddEnvVar("CODEX_LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
+
+            // This makes the node announce itself to its local (pod) IP address.
+            AddEnvVar("NAT_IP_AUTO", "true");
 
             var listenPort = AddInternalPort();
-            AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
+            AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
 
-            if (config.LogLevel != null)
+            if (!string.IsNullOrEmpty(config.BootstrapSpr))
             {
-                AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
+                AddEnvVar("CODEX_BOOTSTRAP_NODE", config.BootstrapSpr);
             }
             if (config.StorageQuota != null)
             {
-                AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
+                AddEnvVar("CODEX_STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);
             }
-            if (config.MetricsEnabled)
+            if (config.BlockTTL != null)
             {
-                AddEnvVar("METRICS_ADDR", "0.0.0.0");
-                AddInternalPortAndVar("METRICS_PORT", tag: MetricsPortTag);
+                AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
+            }
+            if (config.BlockMaintenanceInterval != null)
+            {
+                AddEnvVar("CODEX_BLOCK_MI", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
+            }
+            if (config.BlockMaintenanceNumber != null)
+            {
+                AddEnvVar("CODEX_BLOCK_MN", config.BlockMaintenanceNumber.ToString()!);
+            }
+            if (config.MetricsMode != Metrics.MetricsMode.None)
+            {
+                AddEnvVar("CODEX_METRICS", "true");
+                AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
+                AddInternalPortAndVar("CODEX_METRICS_PORT", tag: MetricsPortTag);
             }
 
             if (config.MarketplaceConfig != null)
             {
                 var gethConfig = startupConfig.Get<GethStartResult>();
-                var companionNode = gethConfig.CompanionNodes[Index];
-                Additional(companionNode);
+                var companionNode = gethConfig.CompanionNode;
+                var companionNodeAccount = companionNode.Accounts[GetAccountIndex(config.MarketplaceConfig)];
+                Additional(companionNodeAccount);
 
-                // Bootstrap node access from within the cluster:
-                //var ip = gethConfig.BootstrapNode.RunningContainers.RunningPod.Ip;
-                //var port = gethConfig.BootstrapNode.RunningContainers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag);
+                var ip = companionNode.RunningContainer.Pod.PodInfo.Ip;
+                var port = companionNode.RunningContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag).Number;
 
-                //AddEnvVar("ETH_PROVIDER", "todo");
-                //AddEnvVar("ETH_ACCOUNT", companionNode.Account);
-                //AddEnvVar("ETH_DEPLOYMENT", "todo");
+                AddEnvVar("CODEX_ETH_PROVIDER", $"ws://{ip}:{port}");
+                AddEnvVar("CODEX_ETH_ACCOUNT", companionNodeAccount.Account);
+                AddEnvVar("CODEX_MARKETPLACE_ADDRESS", gethConfig.MarketplaceNetwork.Marketplace.Address);
+                AddEnvVar("CODEX_PERSISTENCE", "true");
+
+                if (config.MarketplaceConfig.IsValidator)
+                {
+                    AddEnvVar("CODEX_VALIDATOR", "true");
+                }
             }
         }
+
+        private int GetAccountIndex(MarketplaceInitialConfig marketplaceConfig)
+        {
+            if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value;
+            return Index;
+        }
+
+        private string GetDockerImage()
+        {
+            var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
+            if (!string.IsNullOrEmpty(image)) return image;
+            return DefaultDockerImage;
+        }
     }
 }

DistTestCore/Codex/CodexDeployment.cs

@@ -0,0 +1,57 @@
+using DistTestCore.Marketplace;
+using KubernetesWorkflow;
+
+namespace DistTestCore.Codex
+{
+    public class CodexDeployment
+    {
+        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, GrafanaStartInfo? grafanaStartInfo, DeploymentMetadata metadata)
+        {
+            GethStartResult = gethStartResult;
+            CodexContainers = codexContainers;
+            PrometheusContainer = prometheusContainer;
+            GrafanaStartInfo = grafanaStartInfo;
+            Metadata = metadata;
+        }
+
+        public GethStartResult GethStartResult { get; }
+        public RunningContainer[] CodexContainers { get; }
+        public RunningContainer? PrometheusContainer { get; }
+        public GrafanaStartInfo? GrafanaStartInfo { get; }
+        public DeploymentMetadata Metadata { get; }
+    }
+
+    public class DeploymentMetadata
+    {
+        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
+        {
+            DeployDateTimeUtc = DateTime.UtcNow;
+            KubeNamespace = kubeNamespace;
+            NumberOfCodexNodes = numberOfCodexNodes;
+            NumberOfValidators = numberOfValidators;
+            StorageQuotaMB = storageQuotaMB;
+            CodexLogLevel = codexLogLevel;
+            InitialTestTokens = initialTestTokens;
+            MinPrice = minPrice;
+            MaxCollateral = maxCollateral;
+            MaxDuration = maxDuration;
+            BlockTTL = blockTTL;
+            BlockMI = blockMI;
+            BlockMN = blockMN;
+        }
+
+        public DateTime DeployDateTimeUtc { get; }
+        public string KubeNamespace { get; }
+        public int NumberOfCodexNodes { get; }
+        public int NumberOfValidators { get; }
+        public int StorageQuotaMB { get; }
+        public CodexLogLevel CodexLogLevel { get; }
+        public int InitialTestTokens { get; }
+        public int MinPrice { get; }
+        public int MaxCollateral { get; }
+        public int MaxDuration { get; }
+        public int BlockTTL { get; }
+        public int BlockMI { get; }
+        public int BlockMN { get; }
+    }
+}

DistTestCore/Codex/CodexStartupConfig.cs

@@ -1,16 +1,25 @@
 using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
 using KubernetesWorkflow;
 
 namespace DistTestCore.Codex
 {
     public class CodexStartupConfig
     {
-        public Location Location { get; set; }
-        public CodexLogLevel? LogLevel { get; set; }
-        public ByteSize? StorageQuota { get; set; }
-        public bool MetricsEnabled { get; set; }
-        public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
+        public CodexStartupConfig(CodexLogLevel logLevel)
+        {
+            LogLevel = logLevel;
+        }
 
-        //public IOnlineCodexNode? BootstrapNode { get; private set; }
+        public string? NameOverride { get; set; }
+        public Location Location { get; set; }
+        public CodexLogLevel LogLevel { get; }
+        public ByteSize? StorageQuota { get; set; }
+        public MetricsMode MetricsMode { get; set; }
+        public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
+        public string? BootstrapSpr { get; set; }
+        public int? BlockTTL { get; set; }
+        public TimeSpan? BlockMaintenanceInterval { get; set; }
+        public int? BlockMaintenanceNumber { get; set; }
     }
 }

DistTestCore/CodexNodeGroup.cs

@@ -14,12 +14,13 @@ namespace DistTestCore
     {
         private readonly TestLifecycle lifecycle;
 
-        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
         {
             this.lifecycle = lifecycle;
             Setup = setup;
             Containers = containers;
-            Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
+            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
+            Version = new CodexDebugVersionResponse();
         }
 
         public IOnlineCodexNode this[int index]
@@ -44,8 +45,9 @@ namespace DistTestCore
         }
 
         public CodexSetup Setup { get; private set; }
-        public RunningContainers Containers { get; private set; }
+        public RunningContainers[] Containers { get; private set; }
         public OnlineCodexNode[] Nodes { get; private set; }
+        public CodexDebugVersionResponse Version { get; private set; }
 
         public IEnumerator<IOnlineCodexNode> GetEnumerator()
         {
@@ -59,28 +61,28 @@ namespace DistTestCore
 
         public string Describe()
         {
-            return $"<CodexNodeGroup@{Containers.Describe()}-{Setup.Describe()}>";
+            return $"group:[{Containers.Describe()}]";
+        }
+
+        public void EnsureOnline()
+        {
+            foreach (var node in Nodes) node.EnsureOnlineGetVersionResponse();
+            var versionResponses = Nodes.Select(n => n.Version);
+
+            var first = versionResponses.First();
+            if (!versionResponses.All(v => v.version == first.version && v.revision == first.revision))
+            {
+                throw new Exception("Inconsistent version information received from one or more Codex nodes: " +
+                    string.Join(",", versionResponses.Select(v => v.ToString())));
+            }
+
+            Version = first;
         }
 
         private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
         {
-            var access = new CodexAccess(c);
-            EnsureOnline(access);
+            var access = new CodexAccess(lifecycle.Log, c, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(c));
             return factory.CreateOnlineCodexNode(access, this);
         }
-
-        private void EnsureOnline(CodexAccess access)
-        {
-            try
-            {
-                var debugInfo = access.GetDebugInfo();
-                if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
-            }
-            catch (Exception e)
-            {
-                lifecycle.Log.Error($"Failed to start codex node: {e}");
-                throw;
-            }
-        }
     }
 }

DistTestCore/CodexSetup.cs

@@ -6,30 +6,33 @@ namespace DistTestCore
 {
     public interface ICodexSetup
     {
+        ICodexSetup WithName(string name);
         ICodexSetup At(Location location);
-        ICodexSetup WithLogLevel(CodexLogLevel level);
-        //ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node);
+        ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
         ICodexSetup WithStorageQuota(ByteSize storageQuota);
+        ICodexSetup WithBlockTTL(TimeSpan duration);
+        ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration);
+        ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks);
         ICodexSetup EnableMetrics();
-        ICodexSetup EnableMarketplace(int initialBalance);
-        ICodexNodeGroup BringOnline();
+        ICodexSetup EnableMarketplace(TestToken initialBalance);
+        ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
+        ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator);
     }
     
     public class CodexSetup : CodexStartupConfig, ICodexSetup
     {
-        private readonly CodexStarter starter;
-
         public int NumberOfNodes { get; }
 
-        public CodexSetup(CodexStarter starter, int numberOfNodes)
+        public CodexSetup(int numberOfNodes, CodexLogLevel logLevel)
+            : base(logLevel)
         {
-            this.starter = starter;
             NumberOfNodes = numberOfNodes;
         }
 
-        public ICodexNodeGroup BringOnline()
+        public ICodexSetup WithName(string name)
         {
-            return starter.BringOnline(this);
+            NameOverride = name;
+            return this;
         }
 
         public ICodexSetup At(Location location)
@@ -38,15 +41,9 @@ namespace DistTestCore
             return this;
         }
 
-        //public ICodexSetupConfig WithBootstrapNode(IOnlineCodexNode node)
-        //{
-        //    BootstrapNode = node;
-        //    return this;
-        //}
-
-        public ICodexSetup WithLogLevel(CodexLogLevel level)
+        public ICodexSetup WithBootstrapNode(IOnlineCodexNode node)
         {
-            LogLevel = level;
+            BootstrapSpr = node.GetDebugInfo().spr;
             return this;
         }
 
@@ -56,29 +53,57 @@ namespace DistTestCore
             return this;
         }
 
-        public ICodexSetup EnableMetrics()
+        public ICodexSetup WithBlockTTL(TimeSpan duration)
         {
-            MetricsEnabled = true;
+            BlockTTL = Convert.ToInt32(duration.TotalSeconds);
             return this;
         }
 
-        public ICodexSetup EnableMarketplace(int initialBalance)
+        public ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration)
         {
-            MarketplaceConfig = new MarketplaceInitialConfig(initialBalance);
+            BlockMaintenanceInterval = duration;
+            return this;
+        }
+
+        public ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks)
+        {
+            BlockMaintenanceNumber = numberOfBlocks;
+            return this;
+        }
+
+        public ICodexSetup EnableMetrics()
+        {
+            MetricsMode = Metrics.MetricsMode.Record;
+            return this;
+        }
+
+        public ICodexSetup EnableMarketplace(TestToken initialBalance)
+        {
+            return EnableMarketplace(initialBalance, 1000.Eth());
+        }
+
+        public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther)
+        {
+            return EnableMarketplace(initialBalance, initialEther, false);
+        }
+
+        public ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther, bool isValidator)
+        {
+            MarketplaceConfig = new MarketplaceInitialConfig(initialEther, initialBalance, isValidator);
             return this;
         }
 
         public string Describe()
         {
             var args = string.Join(',', DescribeArgs());
-            return $"({NumberOfNodes} CodexNodes with [{args}])";
+            return $"({NumberOfNodes} CodexNodes with args:[{args}])";
         }
 
         private IEnumerable<string> DescribeArgs()
         {
-            if (LogLevel != null) yield return $"LogLevel={LogLevel}";
-            //if (BootstrapNode != null) yield return "BootstrapNode=set-not-shown-here";
-            if (StorageQuota != null) yield return $"StorageQuote={StorageQuota.SizeInBytes}";
+            yield return $"LogLevel={LogLevel}";
+            if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
+            if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
         }
     }
 }

DistTestCore/CodexStarter.cs

@@ -1,85 +1,158 @@
 using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
 using KubernetesWorkflow;
+using Logging;
 
 namespace DistTestCore
 {
-    public class CodexStarter
+    public class CodexStarter : BaseStarter
     {
-        private readonly TestLifecycle lifecycle;
-        private readonly WorkflowCreator workflowCreator;
-
-        public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+        public CodexStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
-            this.lifecycle = lifecycle;
-            this.workflowCreator = workflowCreator;
         }
 
         public List<CodexNodeGroup> RunningGroups { get; } = new List<CodexNodeGroup>();
 
         public ICodexNodeGroup BringOnline(CodexSetup codexSetup)
         {
-            Log($"Starting {codexSetup.Describe()}...");
+            LogSeparator();
+            LogStart($"Starting {codexSetup.Describe()}...");
             var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
 
-            var startupConfig = new StartupConfig();
-            startupConfig.Add(codexSetup);
-            startupConfig.Add(gethStartResult);
+            var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
 
             var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
 
-            var metricAccessFactory = lifecycle.PrometheusStarter.CollectMetricsFor(codexSetup, containers);
+            var metricAccessFactory = CollectMetrics(codexSetup, containers);
             
             var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
 
             var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
-            Log($"Started at '{group.Containers.RunningPod.Ip}'");
+            lifecycle.SetCodexVersion(group.Version);
+
+            var nl = Environment.NewLine;
+            var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
+            LogEnd($"Started {codexSetup.NumberOfNodes} nodes " +
+                $"of image '{containers.Containers().First().Recipe.Image}' " +
+                $"and version '{group.Version}'{nl}" +
+                podInfos);
+            LogSeparator();
+
             return group;
         }
 
         public void BringOffline(CodexNodeGroup group)
         {
-            Log($"Stopping {group.Describe()}...");
+            LogStart($"Stopping {group.Describe()}...");
             var workflow = CreateWorkflow();
-            workflow.Stop(group.Containers);
+            foreach (var c in group.Containers)
+            {
+                StopCrashWatcher(c);
+                workflow.Stop(c);
+            }
             RunningGroups.Remove(group);
-            Log("Stopped.");
+            LogEnd("Stopped.");
         }
 
         public void DeleteAllResources()
         {
             var workflow = CreateWorkflow();
-            workflow.DeleteAllResources();
+            workflow.DeleteTestResources();
 
             RunningGroups.Clear();
         }
 
-        public void DownloadLog(RunningContainer container, ILogHandler logHandler)
+        public void DownloadLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
         {
             var workflow = CreateWorkflow();
-            workflow.DownloadContainerLog(container, logHandler);
-        }
-        
-        private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
-        {
-            var workflow = CreateWorkflow();
-            return workflow.Start(numberOfNodes, location, new CodexContainerRecipe(), startupConfig);
+            workflow.DownloadContainerLog(container, logHandler, tailLines);
         }
 
-        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory)
+        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
+        {
+            if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory();
+
+            var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
+
+            if (codexSetup.MetricsMode == MetricsMode.Dashboard)
+            {
+                lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup);
+            }
+
+            return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
+        }
+
+        private StartupConfig CreateStartupConfig(GethStartResult gethStartResult, CodexSetup codexSetup)
+        {
+            var startupConfig = new StartupConfig();
+            startupConfig.NameOverride = codexSetup.NameOverride;
+            startupConfig.Add(codexSetup);
+            startupConfig.Add(gethStartResult);
+            return startupConfig;
+        }
+
+        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
+        {
+            var result = new List<RunningContainers>();
+            var recipe = new CodexContainerRecipe();
+            for (var i = 0; i < numberOfNodes; i++)
+            {
+                var workflow = CreateWorkflow();
+                var rc = workflow.Start(1, location, recipe, startupConfig);
+                CreateCrashWatcher(workflow, rc);
+                result.Add(rc);
+            }
+            return result.ToArray();
+        }
+
+        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
             RunningGroups.Add(group);
+
+            try
+            {
+                Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
+            }
+            catch
+            {
+                CodexNodesNotOnline(runningContainers);
+                throw;
+            }
+
             return group;
         }
 
+        private void CodexNodesNotOnline(RunningContainers[] runningContainers)
+        {
+            Log("Codex nodes failed to start");
+            foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
+        }
+
         private StartupWorkflow CreateWorkflow()
         {
-            return workflowCreator.CreateWorkflow();
+            return lifecycle.WorkflowCreator.CreateWorkflow();
         }
 
-        private void Log(string msg)
+        private void LogSeparator()
         {
-            lifecycle.Log.Log(msg);
+            Log("----------------------------------------------------------------------------");
+        }
+
+        private void CreateCrashWatcher(StartupWorkflow workflow, RunningContainers rc)
+        {
+            var c = rc.Containers.Single();
+            c.CrashWatcher = workflow.CreateCrashWatcher(c);
+        }
+
+        private void StopCrashWatcher(RunningContainers containers)
+        {
+            foreach (var c in containers.Containers)
+            {
+                c.CrashWatcher?.Stop();
+            }
         }
     }
 }

DistTestCore/Configuration.cs

@@ -1,32 +1,144 @@
-using KubernetesWorkflow;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+using System.Net.NetworkInformation;
+using Utils;
 
 namespace DistTestCore
 {
     public class Configuration
     {
-        public KubernetesWorkflow.Configuration GetK8sConfiguration()
+        private readonly string? kubeConfigFile;
+        private readonly string logPath;
+        private readonly bool logDebug;
+        private readonly string dataFilesPath;
+        private readonly CodexLogLevel codexLogLevel;
+        private readonly string k8sNamespacePrefix;
+        private static RunnerLocation? runnerLocation = null;
+
+        public Configuration()
+        {
+            kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null);
+            logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs");
+            logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
+            dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
+            codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
+            k8sNamespacePrefix = "ct-";
+        }
+
+        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, string k8sNamespacePrefix)
+        {
+            this.kubeConfigFile = kubeConfigFile;
+            this.logPath = logPath;
+            this.logDebug = logDebug;
+            this.dataFilesPath = dataFilesPath;
+            this.codexLogLevel = codexLogLevel;
+            this.k8sNamespacePrefix = k8sNamespacePrefix;
+        }
+
+        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
         {
             return new KubernetesWorkflow.Configuration(
-                k8sNamespace: "codex-test-ns",
-                kubeConfigFile: null,
-                operationTimeout: Timing.K8sOperationTimeout(),
-                retryDelay: Timing.K8sServiceDelay(),
-                locationMap: new[]
-                {
-                    new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),
-                    new ConfigurationLocationEntry(Location.BensLaptop, "worker02"),
-                } 
+                k8sNamespacePrefix: k8sNamespacePrefix,
+                kubeConfigFile: kubeConfigFile,
+                operationTimeout: timeSet.K8sOperationTimeout(),
+                retryDelay: timeSet.WaitForK8sServiceDelay()
             );
         }
 
         public Logging.LogConfig GetLogConfig()
         {
-            return new Logging.LogConfig("D:/CodexTestLogs");
+            return new Logging.LogConfig(logPath, debugEnabled: logDebug);
         }
 
         public string GetFileManagerFolder()
         {
-            return "TestDataFiles";
+            return dataFilesPath;
+        }
+
+        public CodexLogLevel GetCodexLogLevel()
+        {
+            return codexLogLevel;
+        }
+
+        public Address GetAddress(RunningContainer container)
+        {
+            if (runnerLocation == null)
+            {
+                runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container);
+            }
+            
+            if (runnerLocation == RunnerLocation.InternalToCluster)
+            {
+                return container.ClusterInternalAddress;
+            }
+            return container.ClusterExternalAddress;
+        }
+
+        private static string GetEnvVarOrDefault(string varName, string defaultValue)
+        {
+            var v = Environment.GetEnvironmentVariable(varName);
+            if (v == null) return defaultValue;
+            return v;
+        }
+
+        private static string? GetNullableEnvVarOrDefault(string varName, string? defaultValue)
+        {
+            var v = Environment.GetEnvironmentVariable(varName);
+            if (v == null) return defaultValue;
+            return v;
+        }
+    }
+
+    public enum RunnerLocation
+    {
+        ExternalToCluster,
+        InternalToCluster,
+    }
+
+    public static class RunnerLocationUtils
+    {
+        private static bool alreadyDidThat = false;
+
+        public static RunnerLocation DetermineRunnerLocation(RunningContainer container)
+        {
+            // We want to be sure we don't ping more often than strictly necessary.
+            // If we have already determined the location during this application
+            // lifetime, don't do it again.
+            if (alreadyDidThat) throw new Exception("We already did that.");
+            alreadyDidThat = true;
+
+            if (PingHost(container.Pod.PodInfo.Ip))
+            {
+                return RunnerLocation.InternalToCluster;
+            }
+            if (PingHost(Format(container.ClusterExternalAddress)))
+            {
+                return RunnerLocation.ExternalToCluster;
+            }
+
+            throw new Exception("Unable to determine runner location.");
+        }
+
+        private static string Format(Address host)
+        {
+            return host.Host
+                .Replace("http://", "")
+                .Replace("https://", "");
+        }
+
+        private static bool PingHost(string host)
+        {
+            try
+            {
+                using var pinger = new Ping();
+                PingReply reply = pinger.Send(host);
+                return reply.Status == IPStatus.Success;
+            }
+            catch (PingException)
+            {
+            }
+
+            return false;
         }
     }
 }

DistTestCore/DistTest.cs

@@ -1,48 +1,64 @@
 using DistTestCore.Codex;
+using DistTestCore.Helpers;
 using DistTestCore.Logs;
 using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
 using KubernetesWorkflow;
 using Logging;
 using NUnit.Framework;
-using Utils;
+using System.Reflection;
 
 namespace DistTestCore
 {
-    [SetUpFixture]
+    [Parallelizable(ParallelScope.All)]
     public abstract class DistTest
     {
+        private const string TestsType = "dist-tests";
         private readonly Configuration configuration = new Configuration();
-        private FixtureLog fixtureLog = null!;
-        private TestLifecycle lifecycle = null!;
-        private DateTime testStart = DateTime.MinValue;
+        private readonly Assembly[] testAssemblies;
+        private readonly FixtureLog fixtureLog;
+        private readonly StatusLog statusLog;
+        private readonly object lifecycleLock = new object();
+        private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
+
+        public DistTest()
+        {
+            var assemblies = AppDomain.CurrentDomain.GetAssemblies();
+            testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
+
+            var logConfig = configuration.GetLogConfig();
+            var startTime = DateTime.UtcNow;
+            fixtureLog = new FixtureLog(logConfig, startTime);
+            statusLog = new StatusLog(logConfig, startTime);
+        }
 
         [OneTimeSetUp]
         public void GlobalSetup()
         {
+            fixtureLog.Log($"Codex Distributed Tests are starting...");
+            fixtureLog.Log($"Codex image: '{new CodexContainerRecipe().Image}'");
+            fixtureLog.Log($"CodexContracts image: '{new CodexContractsContainerRecipe().Image}'");
+            fixtureLog.Log($"Prometheus image: '{new PrometheusContainerRecipe().Image}'");
+            fixtureLog.Log($"Geth image: '{new GethContainerRecipe().Image}'");
+
             // Previous test run may have been interrupted.
             // Begin by cleaning everything up.
-            fixtureLog = new FixtureLog(configuration.GetLogConfig());
-
             try
             {
                 Stopwatch.Measure(fixtureLog, "Global setup", () =>
                 {
-                    var wc = new WorkflowCreator(configuration.GetK8sConfiguration());
+                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), new PodLabels(TestsType, null!), string.Empty);
                     wc.CreateWorkflow().DeleteAllResources();
                 });                
             }
             catch (Exception ex)
             {
                 GlobalTestFailure.HasFailed = true;
-                Error($"Global setup cleanup failed with: {ex}");
+                fixtureLog.Error($"Global setup cleanup failed with: {ex}");
                 throw;
             }
 
             fixtureLog.Log("Global setup cleanup successful");
-            fixtureLog.Log($"Codex image: {CodexContainerRecipe.DockerImage}");
-            fixtureLog.Log($"Prometheus image: {PrometheusContainerRecipe.DockerImage}");
-            fixtureLog.Log($"Geth image: {GethContainerRecipe.DockerImage}");
         }
 
         [SetUp]
@@ -67,22 +83,173 @@ namespace DistTestCore
             }
             catch (Exception ex)
             {
-                Error("Cleanup failed: " + ex.Message);
+                fixtureLog.Error("Cleanup failed: " + ex.Message);
                 GlobalTestFailure.HasFailed = true;
             }
         }
 
-        public TestFile GenerateTestFile(ByteSize size)
+        public TestFile GenerateTestFile(ByteSize size, string label = "")
         {
-            return lifecycle.FileManager.GenerateTestFile(size);
+            return Get().FileManager.GenerateTestFile(size, label);
         }
 
-        public ICodexSetup SetupCodexNodes(int numberOfNodes)
+        /// <summary>
+        /// Any test files generated in 'action' will be deleted after it returns.
+        /// This helps prevent large tests from filling up discs.
+        /// </summary>
+        public void ScopedTestFiles(Action action)
         {
-            return new CodexSetup(lifecycle.CodexStarter, numberOfNodes);
+            Get().FileManager.PushFileSet();
+            action();
+            Get().FileManager.PopFileSet();
         }
 
-        private void IncludeLogsAndMetricsOnTestFailure()
+        public IOnlineCodexNode SetupCodexBootstrapNode()
+        {
+            return SetupCodexBootstrapNode(s => { });
+        }
+
+        public virtual IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
+        {
+            return SetupCodexNode(s =>
+            {
+                setup(s);
+                s.WithName("Bootstrap");
+            });
+        }
+
+        public IOnlineCodexNode SetupCodexNode()
+        {
+            return SetupCodexNode(s => { });
+        }
+
+        public IOnlineCodexNode SetupCodexNode(Action<ICodexSetup> setup)
+        {
+            return SetupCodexNodes(1, setup)[0];
+        }
+
+        public ICodexNodeGroup SetupCodexNodes(int numberOfNodes)
+        {
+            return SetupCodexNodes(numberOfNodes, s => { });
+        }
+
+        public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
+        {
+            var codexSetup = CreateCodexSetup(numberOfNodes);
+
+            setup(codexSetup);
+
+            return BringOnline(codexSetup);
+        }
+
+        public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
+        {
+            return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
+        }
+
+        public IEnumerable<IOnlineCodexNode> GetAllOnlineCodexNodes()
+        {
+            return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
+        }
+
+        public BaseLog GetTestLog()
+        {
+            return Get().Log;
+        }
+
+        public void Log(string msg)
+        {
+            TestContext.Progress.WriteLine(msg);
+            GetTestLog().Log(msg);
+        }
+
+        public void Debug(string msg)
+        {
+            TestContext.Progress.WriteLine(msg);
+            GetTestLog().Debug(msg);
+        }
+
+        public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers()
+        {
+            return new PeerConnectionTestHelpers(GetTestLog());
+        }
+
+        public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers()
+        {
+            return new PeerDownloadTestHelpers(GetTestLog(), Get().FileManager);
+        }
+
+        public void Measure(string name, Action action)
+        {
+            Stopwatch.Measure(Get().Log, name, action);
+        }
+
+        protected CodexSetup CreateCodexSetup(int numberOfNodes)
+        {
+            return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
+        }
+
+        private TestLifecycle Get()
+        {
+            lock (lifecycleLock)
+            {
+                return lifecycles[GetCurrentTestName()];
+            }
+        }
+
+        private void CreateNewTestLifecycle()
+        {
+            var testName = GetCurrentTestName();
+            fixtureLog.WriteLogTag();
+            Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
+            {
+                lock (lifecycleLock)
+                {
+                    var testNamespace = Guid.NewGuid().ToString();
+                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), TestsType, testNamespace);
+                    lifecycles.Add(testName, lifecycle);
+                }
+            });
+        }
+
+        private void DisposeTestLifecycle()
+        {
+            var lifecycle = Get();
+            var testResult = GetTestResult();
+            var testDuration = lifecycle.GetTestDuration();
+            fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
+            statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetApplicationIds());
+            Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
+            {
+                lifecycle.Log.EndTest();
+                IncludeLogsAndMetricsOnTestFailure(lifecycle);
+                lifecycle.DeleteAllResources();
+                lifecycle = null!;
+            });
+        }
+
+        private ITimeSet GetTimeSet()
+        {
+            if (ShouldUseLongTimeouts()) return new LongTimeSet();
+            return new DefaultTimeSet();
+        }
+
+        private bool ShouldUseLongTimeouts()
+        {
+            // Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
+            // But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
+            // if the attribute is present.
+            var currentTest = TestContext.CurrentContext.Test;
+            var className = currentTest.ClassName;
+            var methodName = currentTest.MethodName;
+
+            var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
+            var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
+
+            return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
+        }
+
+        private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
         {
             var result = TestContext.CurrentContext.Result;
             if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
@@ -91,67 +258,30 @@ namespace DistTestCore
 
                 if (IsDownloadingLogsAndMetricsEnabled())
                 {
-                    Log("Downloading all CodexNode logs and metrics because of test failure...");
-                    DownloadAllLogs();
-                    DownloadAllMetrics();
+                    lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
+                    DownloadAllLogs(lifecycle);
+                    DownloadAllMetrics(lifecycle);
                 }
                 else
                 {
-                    Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
+                    lifecycle.Log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
                 }
             }
         }
 
-        private void Log(string msg)
+        private void DownloadAllLogs(TestLifecycle lifecycle)
         {
-            lifecycle.Log.Log(msg);
-        }
-
-        private void Error(string msg)
-        {
-            lifecycle.Log.Error(msg);
-        }
-
-        private void CreateNewTestLifecycle()
-        {
-            Stopwatch.Measure(fixtureLog, $"Setup for {GetCurrentTestName()}", () =>
+            OnEachCodexNode(lifecycle, node =>
             {
-                lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration);
-                testStart = DateTime.UtcNow;
+                lifecycle.DownloadLog(node.CodexAccess.Container);
             });
         }
 
-        private void DisposeTestLifecycle()
-        {
-            fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({GetTestDuration()})");
-            Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
-            {
-                lifecycle.Log.EndTest();
-                IncludeLogsAndMetricsOnTestFailure();
-                lifecycle.DeleteAllResources();
-                lifecycle = null!;
-            });
-        }
-
-        private string GetTestDuration()
-        {
-            var testDuration = DateTime.UtcNow - testStart;
-            return Time.FormatDuration(testDuration);
-        }
-
-        private void DownloadAllLogs()
-        {
-            OnEachCodexNode(node =>
-            {
-                lifecycle.DownloadLog(node);
-            });
-        }
-
-        private void DownloadAllMetrics()
+        private void DownloadAllMetrics(TestLifecycle lifecycle)
         {
             var metricsDownloader = new MetricsDownloader(lifecycle.Log);
 
-            OnEachCodexNode(node =>
+            OnEachCodexNode(lifecycle, node =>
             {
                 var m = node.Metrics as MetricsAccess;
                 if (m != null)
@@ -161,7 +291,7 @@ namespace DistTestCore
             });
         }
 
-        private void OnEachCodexNode(Action<OnlineCodexNode> action)
+        private void OnEachCodexNode(TestLifecycle lifecycle, Action<OnlineCodexNode> action)
         {
             var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
             foreach (var node in allNodes)

DistTestCore/DistTestCore.csproj

@@ -1,13 +1,26 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>net7.0</TargetFramework>
 	  <RootNamespace>DistTestCore</RootNamespace>
 	  <ImplicitUsings>enable</ImplicitUsings>
 	  <Nullable>enable</Nullable>
+    <IsArm64 Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture)' == 'Arm64'">true</IsArm64>
   </PropertyGroup>
+  <PropertyGroup Condition="'$(IsArm64)'=='true'">
+    <DefineConstants>Arm64</DefineConstants>
+  </PropertyGroup>
+  <ItemGroup>
+    <None Remove="Metrics\dashboard.json" />
+  </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="Metrics\dashboard.json">
+      <CopyToOutputDirectory>Never</CopyToOutputDirectory>
+    </EmbeddedResource>
+  </ItemGroup>
 
   <ItemGroup>
+	  <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
 	  <PackageReference Include="nunit" Version="3.13.3" />
 	  <PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />
 	  <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />

DistTestCore/FileManager.cs

@@ -1,42 +1,49 @@
 using Logging;
 using NUnit.Framework;
+using Utils;
 
 namespace DistTestCore
 {
     public interface IFileManager
     {
-        TestFile CreateEmptyTestFile();
-        TestFile GenerateTestFile(ByteSize size);
+        TestFile CreateEmptyTestFile(string label = "");
+        TestFile GenerateTestFile(ByteSize size, string label = "");
         void DeleteAllTestFiles();
+        void PushFileSet();
+        void PopFileSet();
     }
 
     public class FileManager : IFileManager
     {
-        public const int ChunkSize = 1024 * 1024;
+        public const int ChunkSize = 1024 * 1024 * 100;
+        private static NumberSource folderNumberSource = new NumberSource(0);
         private readonly Random random = new Random();
-        private readonly TestLog log;
+        private readonly BaseLog log;
         private readonly string folder;
+        private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();
 
-        public FileManager(TestLog log, Configuration configuration)
+        public FileManager(BaseLog log, Configuration configuration)
         {
-            folder = configuration.GetFileManagerFolder();
+            folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5"));
 
             EnsureDirectory();
             this.log = log;
         }
 
-        public TestFile CreateEmptyTestFile()
+        public TestFile CreateEmptyTestFile(string label = "")
         {
-            var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin"));
+            var path = Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin");
+            var result = new TestFile(log, path, label);
             File.Create(result.Filename).Close();
+            if (fileSetStack.Any()) fileSetStack.Last().Add(result);
             return result;
         }
 
-        public TestFile GenerateTestFile(ByteSize size)
+        public TestFile GenerateTestFile(ByteSize size, string label)
         {
-            var result = CreateEmptyTestFile();
-            GenerateFileBytes(result, size);
-            log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'.");
+            var sw = Stopwatch.Begin(log);
+            var result = GenerateFile(size, label);
+            sw.End($"Generated file '{result.Describe()}'.");
             return result;
         }
 
@@ -45,14 +52,71 @@ namespace DistTestCore
             DeleteDirectory();
         }
 
+        public void PushFileSet()
+        {
+            fileSetStack.Add(new List<TestFile>());
+        }
+
+        public void PopFileSet()
+        {
+            if (!fileSetStack.Any()) return;
+            var pop = fileSetStack.Last();
+            fileSetStack.Remove(pop);
+
+            foreach (var file in pop)
+            {
+                try
+                {
+                    File.Delete(file.Filename);
+                }
+                catch { }
+            }
+        }
+
+        private TestFile GenerateFile(ByteSize size, string label)
+        {
+            var result = CreateEmptyTestFile(label);
+            CheckSpaceAvailable(result, size);
+
+            GenerateFileBytes(result, size);
+            return result;
+        }
+
+        private void CheckSpaceAvailable(TestFile testFile, ByteSize size)
+        {
+            var file = new FileInfo(testFile.Filename);
+            var drive = new DriveInfo(file.Directory!.Root.FullName);
+
+            var spaceAvailable = drive.TotalFreeSpace;
+
+            if (spaceAvailable < size.SizeInBytes)
+            {
+                var msg = $"Inconclusive: Not enough disk space to perform test. " +
+                    $"{Formatter.FormatByteSize(size.SizeInBytes)} required. " +
+                    $"{Formatter.FormatByteSize(spaceAvailable)} available.";
+
+                log.Log(msg);
+                Assert.Inconclusive(msg);
+            }
+        }
+
         private void GenerateFileBytes(TestFile result, ByteSize size)
         {
             long bytesLeft = size.SizeInBytes;
+            int chunkSize = ChunkSize;
             while (bytesLeft > 0)
             {
-                var length = Math.Min(bytesLeft, ChunkSize);
-                AppendRandomBytesToFile(result, length);
-                bytesLeft -= length;
+                try
+                {
+                    var length = Math.Min(bytesLeft, chunkSize);
+                    AppendRandomBytesToFile(result, length);
+                    bytesLeft -= length;
+                }
+                catch
+                {
+                    chunkSize = chunkSize / 2;
+                    if (chunkSize < 1024) throw;
+                }
             }
         }
 
@@ -77,20 +141,39 @@ namespace DistTestCore
 
     public class TestFile
     {
-        public TestFile(string filename)
+        private readonly BaseLog log;
+
+        public TestFile(BaseLog log, string filename, string label)
         {
+            this.log = log;
             Filename = filename;
+            Label = label;
         }
 
         public string Filename { get; }
-
-        public long GetFileSize()
-        {
-            var info = new FileInfo(Filename);
-            return info.Length;
-        }
+        public string Label { get; }
 
         public void AssertIsEqual(TestFile? actual)
+        {
+            var sw = Stopwatch.Begin(log);
+            try
+            {
+                AssertEqual(actual);
+            }
+            finally
+            {
+                sw.End($"{nameof(TestFile)}.{nameof(AssertIsEqual)}");
+            }
+        }
+
+        public string Describe()
+        {
+            var sizePostfix = $" ({Formatter.FormatByteSize(GetFileSize())})";
+            if (!string.IsNullOrEmpty(Label)) return Label + sizePostfix;
+            return $"'{Filename}'{sizePostfix}";
+        }
+
+        private void AssertEqual(TestFile? actual)
         {
             if (actual == null) Assert.Fail("TestFile is null.");
             if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
@@ -111,10 +194,25 @@ namespace DistTestCore
                 readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
                 readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
 
-                if (readExpected == 0 && readActual == 0) return;
+                if (readExpected == 0 && readActual == 0)
+                {
+                    log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'.");
+                    return;
+                }
+
                 Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length.");
-                CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal.");
+
+                for (var i = 0; i < readActual; i++)
+                {
+                    if (bytesExpected[i] != bytesActual[i]) Assert.Fail("File contents not equal.");
+                }
             }
         }
+
+        private long GetFileSize()
+        {
+            var info = new FileInfo(Filename);
+            return info.Length;
+        }
     }
 }

DistTestCore/GethStarter.cs

@@ -1,80 +1,88 @@
 using DistTestCore.Marketplace;
-using KubernetesWorkflow;
 
 namespace DistTestCore
 {
-    public class GethStarter
+    public class GethStarter : BaseStarter
     {
-        private readonly GethBootstrapNodeCache bootstrapNodeCache;
+        private readonly MarketplaceNetworkCache marketplaceNetworkCache;
         private readonly GethCompanionNodeStarter companionNodeStarter;
-        private readonly TestLifecycle lifecycle;
 
-        public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+        public GethStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
-            bootstrapNodeCache = new GethBootstrapNodeCache(new GethBootstrapNodeStarter(lifecycle, workflowCreator));
-            companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator);
-            this.lifecycle = lifecycle;
+            marketplaceNetworkCache = new MarketplaceNetworkCache(
+                new GethBootstrapNodeStarter(lifecycle),
+                new CodexContractsStarter(lifecycle));
+            companionNodeStarter = new GethCompanionNodeStarter(lifecycle);
         }
 
         public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup)
         {
             if (codexSetup.MarketplaceConfig == null) return CreateMarketplaceUnavailableResult();
 
-            var bootstrapNode = bootstrapNodeCache.Get();
-            var companionNodes = StartCompanionNodes(codexSetup, bootstrapNode);
+            var marketplaceNetwork = marketplaceNetworkCache.Get();
+            var companionNode = StartCompanionNode(codexSetup, marketplaceNetwork);
 
-            TransferInitialBalance(bootstrapNode, codexSetup.MarketplaceConfig.InitialBalance, companionNodes);
+            LogStart("Setting up initial balance...");
+            TransferInitialBalance(marketplaceNetwork, codexSetup.MarketplaceConfig, companionNode);
+            LogEnd($"Initial balance of {codexSetup.MarketplaceConfig.InitialTestTokens} set for {codexSetup.NumberOfNodes} nodes.");
 
-            return CreateGethStartResult(bootstrapNode, companionNodes);
+            return CreateGethStartResult(marketplaceNetwork, companionNode);
         }
 
-        private void TransferInitialBalance(GethBootstrapNodeInfo bootstrapNode, int initialBalance, GethCompanionNodeInfo[] companionNodes)
+        private void TransferInitialBalance(MarketplaceNetwork marketplaceNetwork, MarketplaceInitialConfig marketplaceConfig, GethCompanionNodeInfo companionNode)
         {
-            var interaction = bootstrapNode.StartInteraction(lifecycle.Log);
-            foreach (var node in companionNodes)
-            {
-                interaction.TransferTo(node.Account, initialBalance);
-            }
+            if (marketplaceConfig.InitialTestTokens.Amount == 0) return;
+
+            var interaction = marketplaceNetwork.StartInteraction(lifecycle);
+            var tokenAddress = marketplaceNetwork.Marketplace.TokenAddress;
+
+            var accounts = companionNode.Accounts.Select(a => a.Account).ToArray();
+            interaction.MintTestTokens(accounts, marketplaceConfig.InitialTestTokens.Amount, tokenAddress);
         }
 
-        private GethStartResult CreateGethStartResult(GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo[] companionNodes)
+        private GethStartResult CreateGethStartResult(MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
         {
-            return new GethStartResult(CreateMarketplaceAccessFactory(bootstrapNode), bootstrapNode, companionNodes);
+            return new GethStartResult(CreateMarketplaceAccessFactory(marketplaceNetwork), marketplaceNetwork, companionNode);
         }
 
         private GethStartResult CreateMarketplaceUnavailableResult()
         {
-            return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, Array.Empty<GethCompanionNodeInfo>());
+            return new GethStartResult(new MarketplaceUnavailableAccessFactory(), null!, null!);
         }
 
-        private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(GethBootstrapNodeInfo bootstrapNode)
+        private IMarketplaceAccessFactory CreateMarketplaceAccessFactory(MarketplaceNetwork marketplaceNetwork)
         {
-            return new GethMarketplaceAccessFactory(lifecycle.Log, bootstrapNode!);
+            return new GethMarketplaceAccessFactory(lifecycle, marketplaceNetwork);
         }
 
-        private GethCompanionNodeInfo[] StartCompanionNodes(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode)
+        private GethCompanionNodeInfo StartCompanionNode(CodexSetup codexSetup, MarketplaceNetwork marketplaceNetwork)
         {
-            return companionNodeStarter.StartCompanionNodesFor(codexSetup, bootstrapNode);
+            return companionNodeStarter.StartCompanionNodeFor(codexSetup, marketplaceNetwork);
         }
     }
 
-    public class GethBootstrapNodeCache
+    public class MarketplaceNetworkCache
     {
         private readonly GethBootstrapNodeStarter bootstrapNodeStarter;
-        private GethBootstrapNodeInfo? bootstrapNode;
+        private readonly CodexContractsStarter codexContractsStarter;
+        private MarketplaceNetwork? network;
 
-        public GethBootstrapNodeCache(GethBootstrapNodeStarter bootstrapNodeStarter)
+        public MarketplaceNetworkCache(GethBootstrapNodeStarter bootstrapNodeStarter, CodexContractsStarter codexContractsStarter)
         {
             this.bootstrapNodeStarter = bootstrapNodeStarter;
+            this.codexContractsStarter = codexContractsStarter;
         }
 
-        public GethBootstrapNodeInfo Get()
+        public MarketplaceNetwork Get()
         {
-            if (bootstrapNode == null)
+            if (network == null)
             {
-                bootstrapNode = bootstrapNodeStarter.StartGethBootstrapNode();
+                var bootstrapInfo = bootstrapNodeStarter.StartGethBootstrapNode();
+                var marketplaceInfo = codexContractsStarter.Start(bootstrapInfo);
+                network = new MarketplaceNetwork(bootstrapInfo, marketplaceInfo );
             }
-            return bootstrapNode;
+            return network;
         }
     }
 }

DistTestCore/GrafanaStarter.cs

@@ -0,0 +1,192 @@
+using DistTestCore.Metrics;
+using IdentityModel.Client;
+using KubernetesWorkflow;
+using Newtonsoft.Json;
+using System.Reflection;
+
+namespace DistTestCore
+{
+    public class GrafanaStarter : BaseStarter
+    {
+        private const string StorageQuotaThresholdReplaceToken = "\"<CODEX_STORAGEQUOTA>\"";
+        private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"<CODEX_BYTESUSED_SOFTMAX>\"";
+
+        public GrafanaStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
+        {
+        }
+
+        public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup)
+        {
+            LogStart($"Starting dashboard server");
+
+            var grafanaContainer = StartGrafanaContainer();
+            var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer);
+
+            var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth);
+
+            Log("Connecting datasource...");
+            AddDataSource(http, prometheusContainer);
+
+            Log("Uploading dashboard configurations...");
+            var jsons = ReadEachDashboardJsonFile(codexSetup);
+            var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
+
+            LogEnd("Dashboard server started.");
+
+            return new GrafanaStartInfo(dashboardUrls, grafanaContainer);
+        }
+
+        private RunningContainer StartGrafanaContainer()
+        {
+            var startupConfig = new StartupConfig();
+
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig);
+            if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created.");
+
+            return grafanaContainers.Containers.First();
+        }
+
+        private void AddBasicAuth(HttpClient client)
+        {
+            client.SetBasicAuthentication(
+                GrafanaContainerRecipe.DefaultAdminUser,
+                GrafanaContainerRecipe.DefaultAdminPassword);
+        }
+
+        private static void AddDataSource(Http http, RunningContainer prometheusContainer)
+        {
+            var prometheusAddress = prometheusContainer.ClusterExternalAddress;
+            var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port;
+            var response = http.HttpPostJson<GrafanaDataSourceRequest, GrafanaDataSourceResponse>("datasources", new GrafanaDataSourceRequest
+            {
+                uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb",
+                name = "CodexPrometheus",
+                type = "prometheus",
+                url = prometheusUrl,
+                access = "proxy",
+                basicAuth = false,
+                jsonData = new GrafanaDataSourceJsonData
+                {
+                    httpMethod = "POST"
+                }
+            });
+
+            if (response.message != "Datasource added")
+            {
+                throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message);
+            }
+        }
+
+        public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson)
+        {
+            var request = GetDashboardCreateRequest(dashboardJson);
+            var response = http.HttpPostString("dashboards/db", request);
+            var jsonResponse = JsonConvert.DeserializeObject<GrafanaPostDashboardResponse>(response);
+            if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard.");
+
+            var grafanaAddress = grafanaContainer.ClusterExternalAddress;
+            return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
+        }
+
+        private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup)
+        {
+            var assembly = Assembly.GetExecutingAssembly();
+            var resourceNames = new[]
+            {
+                "DistTestCore.Metrics.dashboard.json"
+            };
+
+            return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray();
+        }
+
+        private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup)
+        {
+            using var stream = assembly.GetManifestResourceStream(resourceName);
+            if (stream == null) throw new Exception("Unable to find resource " + resourceName);
+            using var reader = new StreamReader(stream);
+            return ApplyReplacements(reader.ReadToEnd(), codexSetup);
+        }
+
+        private static string ApplyReplacements(string input, CodexSetup codexSetup)
+        {
+            var quotaString = GetQuotaString(codexSetup);
+            var softMaxString = GetSoftMaxString(codexSetup);
+
+            return input
+                .Replace(StorageQuotaThresholdReplaceToken, quotaString)
+                .Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString);
+        }
+
+        private static string GetQuotaString(CodexSetup codexSetup)
+        {
+            return GetCodexStorageQuotaInBytes(codexSetup).ToString();
+        }
+
+        private static string GetSoftMaxString(CodexSetup codexSetup)
+        {
+            var quota = GetCodexStorageQuotaInBytes(codexSetup);
+            var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing.
+            return softMax.ToString();
+        }
+
+        private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup)
+        {
+            if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes;
+
+            // Codex default: 8GB
+            return 8.GB().SizeInBytes;
+        }
+
+        private static string GetDashboardCreateRequest(string dashboardJson)
+        {
+            return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}";
+        }
+    }
+
+    public class GrafanaStartInfo
+    {
+        public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container)
+        {
+            DashboardUrls = dashboardUrls;
+            Container = container;
+        }
+
+        public string[] DashboardUrls { get; }
+        public RunningContainer Container { get; }
+    }
+
+    public class GrafanaDataSourceRequest
+    {
+        public string uid { get; set; } = string.Empty;
+        public string name { get; set; } = string.Empty;
+        public string type { get; set; } = string.Empty;
+        public string url { get; set; } = string.Empty;
+        public string access { get; set; } = string.Empty;
+        public bool basicAuth { get; set; }
+        public GrafanaDataSourceJsonData jsonData { get; set; } = new();
+    }
+
+    public class GrafanaDataSourceResponse
+    {
+        public int id { get; set; }
+        public string message { get; set; } = string.Empty;
+        public string name { get; set; } = string.Empty;
+    }
+
+    public class GrafanaDataSourceJsonData
+    {
+        public string httpMethod { get; set; } = string.Empty;
+    }
+
+    public class GrafanaPostDashboardResponse
+    {
+        public int id { get; set; }
+        public string slug { get; set; } = string.Empty;
+        public string status { get; set; } = string.Empty;
+        public string uid { get; set; } = string.Empty;
+        public string url { get; set; } = string.Empty;
+        public int version { get; set; }
+    }
+}

DistTestCore/Helpers/AssertHelpers.cs

@@ -0,0 +1,22 @@
+using NUnit.Framework.Constraints;
+using NUnit.Framework;
+using Utils;
+
+namespace DistTestCore.Helpers
+{
+    public static class AssertHelpers
+    {
+        public static void RetryAssert<T>(IResolveConstraint constraint, Func<T> actual, string message)
+        {
+            try
+            {
+                var c = constraint.Resolve();
+                Time.WaitUntil(() => c.ApplyTo(actual()).IsSuccess);
+            }
+            catch (TimeoutException)
+            {
+                Assert.That(actual(), constraint, message);
+            }
+        }
+    }
+}

DistTestCore/Helpers/FullConnectivityHelper.cs

@@ -0,0 +1,217 @@
+using DistTestCore.Codex;
+using Logging;
+using NUnit.Framework;
+using Utils;
+
+namespace DistTestCore.Helpers
+{
+    public interface IFullConnectivityImplementation
+    {
+        string Description();
+        string ValidateEntry(FullConnectivityHelper.Entry entry, FullConnectivityHelper.Entry[] allEntries);
+        FullConnectivityHelper.PeerConnectionState Check(FullConnectivityHelper.Entry from, FullConnectivityHelper.Entry to);
+    }
+
+    public class FullConnectivityHelper
+    {
+        private static string Nl = Environment.NewLine;
+        private readonly BaseLog log;
+        private readonly IFullConnectivityImplementation implementation;
+
+        public FullConnectivityHelper(BaseLog log, IFullConnectivityImplementation implementation)
+        {
+            this.log = log;
+            this.implementation = implementation;
+        }
+
+        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
+        {
+            AssertFullyConnected(nodes.ToArray());
+        }
+
+        private void AssertFullyConnected(CodexAccess[] nodes)
+        {
+            Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            var entries = CreateEntries(nodes);
+            var pairs = CreatePairs(entries);
+
+            RetryWhilePairs(pairs, () =>
+            {
+                CheckAndRemoveSuccessful(pairs);
+            });
+
+            if (pairs.Any())
+            {
+                var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
+
+                Log($"Connections failed:{Nl}{pairDetails}");
+
+                Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
+            }
+            else
+            {
+                Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
+            }
+        }
+
+        private static void RetryWhilePairs(List<Pair> pairs, Action action)
+        {
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2);
+            while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
+            {
+                action();
+
+                Time.Sleep(TimeSpan.FromSeconds(2));
+            }
+        }
+
+        private void CheckAndRemoveSuccessful(List<Pair> pairs)
+        {
+            // For large sets, don't try and do all of them at once.
+            var selectedPair = pairs.Take(20).ToArray();
+            var pairDetails = new List<string>();
+
+            foreach (var pair in selectedPair)
+            {
+                pair.Check();
+
+                if (pair.Success)
+                {
+                    pairDetails.AddRange(pair.GetResultMessages());
+                    pairs.Remove(pair);
+                }
+            }
+
+            Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
+        }
+
+        private Entry[] CreateEntries(CodexAccess[] nodes)
+        {
+            var entries = nodes.Select(n => new Entry(n)).ToArray();
+
+            var errors = entries
+                            .Select(e => implementation.ValidateEntry(e, entries))
+                            .Where(s => !string.IsNullOrEmpty(s))
+                            .ToArray();
+
+            if (errors.Any())
+            {
+                Assert.Fail("Some node entries failed to validate: " + string.Join(Nl, errors));
+            }
+
+            return entries;
+        }
+
+        private List<Pair> CreatePairs(Entry[] entries)
+        {
+            return CreatePairsIterator(entries).ToList();
+        }
+
+        private IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
+        {
+            for (var x = 0; x < entries.Length; x++)
+            {
+                for (var y = x + 1; y < entries.Length; y++)
+                {
+                    yield return new Pair(implementation, entries[x], entries[y]);
+                }
+            }
+        }
+
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+
+        public class Entry
+        {
+            public Entry(CodexAccess node)
+            {
+                Node = node;
+                Response = node.GetDebugInfo();
+            }
+
+            public CodexAccess Node { get; }
+            public CodexDebugResponse Response { get; }
+
+            public override string ToString()
+            {
+                if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
+                return Response.id;
+            }
+        }
+
+        public enum PeerConnectionState
+        {
+            Unknown,
+            Connection,
+            NoConnection,
+        }
+
+        public class Pair
+        {
+            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
+            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
+            private readonly IFullConnectivityImplementation implementation;
+
+            public Pair(IFullConnectivityImplementation implementation, Entry a, Entry b)
+            {
+                this.implementation = implementation;
+                A = a;
+                B = b;
+            }
+
+            public Entry A { get; }
+            public Entry B { get; }
+            public PeerConnectionState AKnowsB { get; private set; }
+            public PeerConnectionState BKnowsA { get; private set; }
+            public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
+            public bool Inconclusive { get { return AKnowsB == PeerConnectionState.Unknown || BKnowsA == PeerConnectionState.Unknown; } }
+
+            public void Check()
+            {
+                aToBTime = Measure(() => AKnowsB = Check(A, B));
+                bToATime = Measure(() => BKnowsA = Check(B, A));
+            }
+
+            public override string ToString()
+            {
+                return $"[{string.Join(",", GetResultMessages())}]";
+            }
+
+            public string[] GetResultMessages()
+            {
+                var aName = A.ToString();
+                var bName = B.ToString();
+
+                return new[]
+                {
+                    $"[{aName} --> {bName}] = {AKnowsB} ({aToBTime.TotalSeconds} seconds)",
+                    $"[{aName} <-- {bName}] = {BKnowsA} ({bToATime.TotalSeconds} seconds)"
+                };
+            }
+
+            private static TimeSpan Measure(Action action)
+            {
+                var start = DateTime.UtcNow;
+                action();
+                return DateTime.UtcNow - start;
+            }
+
+            private PeerConnectionState Check(Entry from, Entry to)
+            {
+                Thread.Sleep(10);
+
+                try
+                {
+                    return implementation.Check(from, to);
+                }
+                catch
+                {
+                    // Didn't get a conclusive answer. Try again later.
+                    return PeerConnectionState.Unknown;
+                }
+            }
+        }
+    }
+}

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -0,0 +1,71 @@
+using DistTestCore.Codex;
+using Logging;
+using static DistTestCore.Helpers.FullConnectivityHelper;
+
+namespace DistTestCore.Helpers
+{
+    public class PeerConnectionTestHelpers : IFullConnectivityImplementation
+    {
+        private readonly FullConnectivityHelper helper;
+
+        public PeerConnectionTestHelpers(BaseLog log)
+        {
+            helper = new FullConnectivityHelper(log, this);
+        }
+
+        public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullyConnected(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess));
+        }
+
+        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
+        {
+            helper.AssertFullyConnected(nodes);
+        }
+
+        public string Description()
+        {
+            return "Peer Discovery";
+        }
+
+        public string ValidateEntry(Entry entry, Entry[] allEntries)
+        {
+            var result = string.Empty;
+            foreach (var peer in entry.Response.table.nodes)
+            {
+                var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
+                if (expected != peer.address)
+                {
+                    result += $"Node:{entry.Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'. ";
+                }
+            }
+            return result;
+        }
+
+        public PeerConnectionState Check(Entry from, Entry to)
+        {
+            var peerId = to.Response.id;
+
+            var response = from.Node.GetDebugPeer(peerId);
+            if (!response.IsPeerFound)
+            {
+                return PeerConnectionState.NoConnection;
+            }
+            if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
+            {
+                return PeerConnectionState.Connection;
+            }
+            return PeerConnectionState.Unknown;
+        }
+
+        private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
+        {
+            var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
+            if (peer == null) return $"peerId: {node.peerId} is not known.";
+
+            var ip = peer.Node.Container.Pod.PodInfo.Ip;
+            var discPort = peer.Node.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
+            return $"{ip}:{discPort.Number}";
+        }
+    }
+}

DistTestCore/Helpers/PeerDownloadTestHelpers.cs

@@ -0,0 +1,88 @@
+using DistTestCore.Codex;
+using Logging;
+using static DistTestCore.Helpers.FullConnectivityHelper;
+
+namespace DistTestCore.Helpers
+{
+    public class PeerDownloadTestHelpers : IFullConnectivityImplementation
+    {
+        private readonly FullConnectivityHelper helper;
+        private readonly BaseLog log;
+        private readonly FileManager fileManager;
+        private ByteSize testFileSize;
+
+        public PeerDownloadTestHelpers(BaseLog log, FileManager fileManager)
+        {
+            helper = new FullConnectivityHelper(log, this);
+            testFileSize = 1.MB();
+            this.log = log;
+            this.fileManager = fileManager;
+        }
+
+        public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
+        {
+            AssertFullDownloadInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess), testFileSize);
+        }
+
+        public void AssertFullDownloadInterconnectivity(IEnumerable<CodexAccess> nodes, ByteSize testFileSize)
+        {
+            this.testFileSize = testFileSize;
+            helper.AssertFullyConnected(nodes);
+        }
+
+        public string Description()
+        {
+            return "Download Connectivity";
+        }
+
+        public string ValidateEntry(Entry entry, Entry[] allEntries)
+        {
+            return string.Empty;
+        }
+
+        public PeerConnectionState Check(Entry from, Entry to)
+        {
+            fileManager.PushFileSet();
+            var expectedFile = GenerateTestFile(from.Node, to.Node);
+
+            using var uploadStream = File.OpenRead(expectedFile.Filename);
+            var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(uploadStream));
+
+            try
+            {
+                var downloadedFile = Stopwatch.Measure(log, "Download", () => DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded"));
+                expectedFile.AssertIsEqual(downloadedFile);
+                return PeerConnectionState.Connection;
+            }
+            catch
+            {
+                // Should an exception occur during the download or file-content assertion,
+                // We consider that as no-connection for the purpose of this test.
+                return PeerConnectionState.NoConnection;
+            }
+            finally
+            {
+                fileManager.PopFileSet();
+            }
+
+            // Should an exception occur during upload, then this try is inconclusive and we try again next loop.
+        }
+
+        private TestFile DownloadFile(CodexAccess node, string contentId, string label)
+        {
+            var downloadedFile = fileManager.CreateEmptyTestFile(label);
+            using var downloadStream = File.OpenWrite(downloadedFile.Filename);
+            using var stream = node.DownloadFile(contentId);
+            stream.CopyTo(downloadStream);
+            return downloadedFile;
+        }
+
+        private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader)
+        {
+            var up = uploader.GetName().Replace("<", "").Replace(">", "");
+            var down = downloader.GetName().Replace("<", "").Replace(">", "");
+            var label = $"~from:{up}-to:{down}~";
+            return fileManager.GenerateTestFile(testFileSize, label);
+        }
+    }
+}

DistTestCore/Http.cs

@@ -1,22 +1,33 @@
-using Newtonsoft.Json;
-using NUnit.Framework;
+using Logging;
+using Newtonsoft.Json;
 using System.Net.Http.Headers;
+using System.Net.Http.Json;
 using Utils;
 
 namespace DistTestCore
 {
     public class Http
     {
-        private readonly string ip;
-        private readonly int port;
+        private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
+        private readonly Address address;
         private readonly string baseUrl;
+        private readonly Action<HttpClient> onClientCreated;
+        private readonly string? logAlias;
 
-        public Http(string ip, int port, string baseUrl)
+        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
+            : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
         {
-            this.ip = ip;
-            this.port = port;
-            this.baseUrl = baseUrl;
+        }
 
+        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
+        {
+            this.log = log;
+            this.timeSet = timeSet;
+            this.address = address;
+            this.baseUrl = baseUrl;
+            this.onClientCreated = onClientCreated;
+            this.logAlias = logAlias;
             if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
             if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
         }
@@ -27,14 +38,55 @@ namespace DistTestCore
             {
                 using var client = GetClient();
                 var url = GetUrl() + route;
+                Log(url, "");
                 var result = Time.Wait(client.GetAsync(url));
-                return Time.Wait(result.Content.ReadAsStringAsync());
-            });
+                var str = Time.Wait(result.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str; ;
+            }, $"HTTP-GET:{route}");
         }
 
         public T HttpGetJson<T>(string route)
         {
-            return JsonConvert.DeserializeObject<T>(HttpGetString(route))!;
+            var json = HttpGetString(route);
+            return TryJsonDeserialize<T>(json);
+        }
+
+        public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
+        {
+            var json = HttpPostJson(route, body);
+            return TryJsonDeserialize<TResponse>(json);
+        }
+
+        public string HttpPostJson<TRequest>(string route, TRequest body)
+        {
+            return Retry(() =>
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                using var content = JsonContent.Create(body);
+                Log(url, JsonConvert.SerializeObject(body));
+                var result = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(result.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-JSON: {route}");
+        }
+
+        public string HttpPostString(string route, string body)
+        {
+            return Retry(() =>
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                Log(url, body);
+                var content = new StringContent(body);
+                content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+                var result = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(result.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STRING: {route}");
         }
 
         public string HttpPostStream(string route, Stream stream)
@@ -43,13 +95,14 @@ namespace DistTestCore
             {
                 using var client = GetClient();
                 var url = GetUrl() + route;
-
+                Log(url, "~ STREAM ~");
                 var content = new StreamContent(stream);
                 content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
                 var response = Time.Wait(client.PostAsync(url, content));
-
-                return Time.Wait(response.Content.ReadAsStringAsync());
-            });
+                var str = Time.Wait(response.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STREAM: {route}");
         }
 
         public Stream HttpGetStream(string route)
@@ -58,44 +111,56 @@ namespace DistTestCore
             {
                 var client = GetClient();
                 var url = GetUrl() + route;
-
+                Log(url, "~ STREAM ~");
                 return Time.Wait(client.GetStreamAsync(url));
-            });
+            }, $"HTTP-GET-STREAM: {route}");
+        }
+
+        public T TryJsonDeserialize<T>(string json)
+        {
+            try
+            {
+                return JsonConvert.DeserializeObject<T>(json)!;
+            }
+            catch (Exception exception)
+            {
+                var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}";
+                throw new InvalidOperationException(msg, exception);
+            }
         }
 
         private string GetUrl()
         {
-            return $"http://{ip}:{port}{baseUrl}";
+            return $"{address.Host}:{address.Port}{baseUrl}";
         }
 
-        private static T Retry<T>(Func<T> operation)
+        private void Log(string url, string message)
         {
-            var retryCounter = 0;
-
-            while (true)
+            if (logAlias != null)
             {
-                try
-                {
-                    return operation();
-                }
-                catch (Exception exception)
-                {
-                    Timing.HttpCallRetryDelay();
-                    retryCounter++;
-                    if (retryCounter > Timing.HttpCallRetryCount())
-                    {
-                        Assert.Fail(exception.Message);
-                        throw;
-                    }
-                }
+                log.Debug($"({logAlias})({url}) = '{message}'", 3);
+            }
+            else
+            {
+                log.Debug($"({url}) = '{message}'", 3);
             }
         }
 
-        private static HttpClient GetClient()
+        private T Retry<T>(Func<T> operation, string description)
+        {
+            return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
+        }
+
+        private HttpClient GetClient()
         {
             var client = new HttpClient();
-            client.Timeout = Timing.HttpCallTimeout();
+            client.Timeout = timeSet.HttpCallTimeout();
+            onClientCreated(client);
             return client;
         }
+
+        private static void DoNothing(HttpClient client)
+        {
+        }
     }
 }

DistTestCore/Logs/CodexNodeLog.cs

@@ -1,35 +0,0 @@
-using Logging;
-using NUnit.Framework;
-
-namespace DistTestCore.Logs
-{
-    public interface ICodexNodeLog
-    {
-        void AssertLogContains(string expectedString);
-    }
-
-    public class CodexNodeLog : ICodexNodeLog
-    {
-        private readonly LogFile logFile;
-
-        public CodexNodeLog(LogFile logFile)
-        {
-            this.logFile = logFile;
-        }
-
-        public void AssertLogContains(string expectedString)
-        {
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (line.Contains(expectedString)) return;
-                line = streamReader.ReadLine();
-            }
-
-            Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
-        }
-    }
-}

DistTestCore/Logs/DownloadedLog.cs

@@ -0,0 +1,64 @@
+using Logging;
+using NUnit.Framework;
+
+namespace DistTestCore.Logs
+{
+    public interface IDownloadedLog
+    {
+        void AssertLogContains(string expectedString);
+        string[] FindLinesThatContain(params string[] tags);
+        void DeleteFile();
+    }
+
+    public class DownloadedLog : IDownloadedLog
+    {
+        private readonly LogFile logFile;
+        private readonly string owner;
+
+        public DownloadedLog(LogFile logFile, string owner)
+        {
+            this.logFile = logFile;
+            this.owner = owner;
+        }
+
+        public void AssertLogContains(string expectedString)
+        {
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                if (line.Contains(expectedString)) return;
+                line = streamReader.ReadLine();
+            }
+
+            Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
+        }
+
+        public string[] FindLinesThatContain(params string[] tags)
+        {
+            var result = new List<string>();
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                if (tags.All(line.Contains))
+                {
+                    result.Add(line);
+                }
+
+                line = streamReader.ReadLine();
+            }
+
+            return result.ToArray();
+        }
+
+        public void DeleteFile()
+        {
+            File.Delete(logFile.FullFilename);
+        }
+    }
+}

DistTestCore/Logs/LogDownloadHandler.cs

@@ -5,19 +5,21 @@ namespace DistTestCore.Logs
 {
     public class LogDownloadHandler : LogHandler, ILogHandler
     {
+        private readonly RunningContainer container;
         private readonly LogFile log;
 
-        public LogDownloadHandler(string description, LogFile log)
+        public LogDownloadHandler(RunningContainer container, string description, LogFile log)
         {
+            this.container = container;
             this.log = log;
 
             log.Write($"{description} -->> {log.FullFilename}");
             log.WriteRaw(description);
         }
 
-        public CodexNodeLog CreateCodexNodeLog()
+        public DownloadedLog DownloadLog()
         {
-            return new CodexNodeLog(log);
+            return new DownloadedLog(log, container.Name);
         }
 
         protected override void ProcessLine(string line)

DistTestCore/Marketplace/CodexContractsContainerConfig.cs

@@ -0,0 +1,16 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class CodexContractsContainerConfig
+    {
+        public CodexContractsContainerConfig(string bootstrapNodeIp, Port jsonRpcPort)
+        {
+            BootstrapNodeIp = bootstrapNodeIp;
+            JsonRpcPort = jsonRpcPort;
+        }
+
+        public string BootstrapNodeIp { get; }
+        public Port JsonRpcPort { get; }
+    }
+}

DistTestCore/Marketplace/CodexContractsContainerRecipe.cs

@@ -0,0 +1,25 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class CodexContractsContainerRecipe : ContainerRecipeFactory
+    {
+        public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
+        public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
+
+        public override string AppName => "codex-contracts";
+        public override string Image => "codexstorage/codex-contracts-eth:latest-dist-tests";
+
+        protected override void Initialize(StartupConfig startupConfig)
+        {
+            var config = startupConfig.Get<CodexContractsContainerConfig>();
+
+            var ip = config.BootstrapNodeIp;
+            var port = config.JsonRpcPort.Number;
+
+            AddEnvVar("DISTTEST_NETWORK_URL", $"http://{ip}:{port}");
+            AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork");
+            AddEnvVar("KEEP_ALIVE", "1");
+        }
+    }
+}

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -0,0 +1,103 @@
+using KubernetesWorkflow;
+using Utils;
+
+namespace DistTestCore.Marketplace
+{
+    public class CodexContractsStarter : BaseStarter
+    {
+
+        public CodexContractsStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
+        {
+        }
+
+        public MarketplaceInfo Start(GethBootstrapNodeInfo bootstrapNode)
+        {
+            LogStart("Deploying Codex Marketplace...");
+
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]);
+
+            var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig);
+            if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure.");
+            var container = containers.Containers[0];
+
+            WaitUntil(() =>
+            {
+                var logHandler = new ContractsReadyLogHandler(Debug);
+                workflow.DownloadContainerLog(container, logHandler, null);
+                return logHandler.Found;
+            });
+            Log("Contracts deployed. Extracting addresses...");
+
+            var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, container);
+            var marketplaceAddress = extractor.ExtractMarketplaceAddress();
+            var abi = extractor.ExtractMarketplaceAbi();
+
+            var interaction = bootstrapNode.StartInteraction(lifecycle);
+            var tokenAddress = interaction.GetTokenAddress(marketplaceAddress);
+
+            LogEnd("Extract completed. Marketplace deployed.");
+
+            return new MarketplaceInfo(marketplaceAddress, abi, tokenAddress);
+        }
+
+        private void WaitUntil(Func<bool> predicate)
+        {
+            Time.WaitUntil(predicate, TimeSpan.FromMinutes(3), TimeSpan.FromSeconds(2));
+        }
+
+        private StartupConfig CreateStartupConfig(RunningContainer bootstrapContainer)
+        {
+            var startupConfig = new StartupConfig();
+            var contractsConfig = new CodexContractsContainerConfig(bootstrapContainer.Pod.PodInfo.Ip, bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.HttpPortTag));
+            startupConfig.Add(contractsConfig);
+            return startupConfig;
+        }
+    }
+
+    public class MarketplaceInfo
+    {
+        public MarketplaceInfo(string address, string abi, string tokenAddress)
+        {
+            Address = address;
+            Abi = abi;
+            TokenAddress = tokenAddress;
+        }
+
+        public string Address { get; }
+        public string Abi { get; }
+        public string TokenAddress { get; }
+    }
+
+    public class ContractsReadyLogHandler : LogHandler
+    {
+        // Log should contain 'Compiled 15 Solidity files successfully' at some point.
+        private const string RequiredCompiledString = "Solidity files successfully";
+        // When script is done, it prints the ready-string.
+        private const string ReadyString = "Done! Sleeping indefinitely...";
+        private readonly Action<string> debug;
+
+        public ContractsReadyLogHandler(Action<string> debug)
+        {
+            this.debug = debug;
+            debug($"Looking for '{RequiredCompiledString}' and '{ReadyString}' in container logs...");
+        }
+
+        public bool SeenCompileString { get; private set; }
+        public bool Found { get; private set; }
+
+        protected override void ProcessLine(string line)
+        {
+            debug(line);
+            if (line.Contains(RequiredCompiledString)) SeenCompileString = true;
+            if (line.Contains(ReadyString))
+            {
+                if (!SeenCompileString) throw new Exception("CodexContracts deployment failed. " +
+                    "Solidity files not compiled before process exited.");
+
+                Found = true;
+            }
+        }
+    }
+}

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -0,0 +1,149 @@
+using KubernetesWorkflow;
+using Logging;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Utils;
+
+namespace DistTestCore.Marketplace
+{
+    public class ContainerInfoExtractor
+    {
+        private readonly BaseLog log;
+        private readonly StartupWorkflow workflow;
+        private readonly RunningContainer container;
+
+        public ContainerInfoExtractor(BaseLog log, StartupWorkflow workflow, RunningContainer container)
+        {
+            this.log = log;
+            this.workflow = workflow;
+            this.container = container;
+        }
+
+        public AllGethAccounts ExtractAccounts()
+        {
+            log.Debug();
+            var accountsCsv = Retry(() => FetchAccountsCsv());
+            if (string.IsNullOrEmpty(accountsCsv)) throw new InvalidOperationException("Unable to fetch accounts.csv for geth node. Test infra failure.");
+
+            var lines = accountsCsv.Split('\n');
+            return new AllGethAccounts(lines.Select(ParseLineToAccount).ToArray());
+        }
+
+        public string ExtractPubKey()
+        {
+            log.Debug();
+            var pubKey = Retry(FetchPubKey);
+            if (string.IsNullOrEmpty(pubKey)) throw new InvalidOperationException("Unable to fetch enode from geth node. Test infra failure.");
+
+            return pubKey;
+        }
+
+        public string ExtractMarketplaceAddress()
+        {
+            log.Debug();
+            var marketplaceAddress = Retry(FetchMarketplaceAddress);
+            if (string.IsNullOrEmpty(marketplaceAddress)) throw new InvalidOperationException("Unable to fetch marketplace account from codex-contracts node. Test infra failure.");
+
+            return marketplaceAddress;
+        }
+
+        public string ExtractMarketplaceAbi()
+        {
+            log.Debug();
+            var marketplaceAbi = Retry(FetchMarketplaceAbi);
+            if (string.IsNullOrEmpty(marketplaceAbi)) throw new InvalidOperationException("Unable to fetch marketplace artifacts from codex-contracts node. Test infra failure.");
+
+            return marketplaceAbi;
+        }
+
+        private string FetchAccountsCsv()
+        {
+            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename);
+        }
+
+        private string FetchMarketplaceAddress()
+        {
+            var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceAddressFilename);
+            var marketplace = JsonConvert.DeserializeObject<MarketplaceJson>(json);
+            return marketplace!.address;
+        }
+
+        private string FetchMarketplaceAbi()
+        {
+            var json = workflow.ExecuteCommand(container, "cat", CodexContractsContainerRecipe.MarketplaceArtifactFilename);
+
+            var artifact = JObject.Parse(json);
+            var abi = artifact["abi"];
+            return abi!.ToString(Formatting.None);
+        }
+
+        private string FetchPubKey()
+        {
+            var enodeFinder = new PubKeyFinder(s => log.Debug(s));
+            workflow.DownloadContainerLog(container, enodeFinder, null);
+            return enodeFinder.GetPubKey();
+        }
+
+        private GethAccount ParseLineToAccount(string l)
+        {
+            var tokens = l.Replace("\r", "").Split(',');
+            if (tokens.Length != 2) throw new InvalidOperationException();
+            var account = tokens[0];
+            var privateKey = tokens[1];
+            return new GethAccount(account, privateKey);
+        }
+
+        private static string Retry(Func<string> fetch)
+        {
+            return Time.Retry(fetch, nameof(ContainerInfoExtractor));
+        }
+    }
+
+    public class PubKeyFinder : LogHandler, ILogHandler
+    {
+        private const string openTag = "self=enode://";
+        private const string openTagQuote = "self=\"enode://";
+        private readonly Action<string> debug;
+        private string pubKey = string.Empty;
+
+        public PubKeyFinder(Action<string> debug)
+        {
+            this.debug = debug;
+            debug($"Looking for '{openTag}' in container logs...");
+        }
+
+        public string GetPubKey()
+        {
+            if (string.IsNullOrEmpty(pubKey)) throw new Exception("Not found yet exception.");
+            return pubKey;
+        }
+
+        protected override void ProcessLine(string line)
+        {
+            debug(line);
+            if (line.Contains(openTag))
+            {
+                ExtractPubKey(openTag, line);
+            }
+            else if (line.Contains(openTagQuote))
+            {
+                ExtractPubKey(openTagQuote, line);
+            }
+        }
+
+        private void ExtractPubKey(string tag, string line)
+        {
+            var openIndex = line.IndexOf(tag) + tag.Length;
+            var closeIndex = line.IndexOf("@");
+
+            pubKey = line.Substring(
+                    startIndex: openIndex,
+                    length: closeIndex - openIndex);
+        }
+    }
+
+    public class MarketplaceJson
+    {
+        public string address { get; set; } = string.Empty;
+    }
+}

DistTestCore/Marketplace/GethBootstrapNodeInfo.cs

@@ -1,33 +1,42 @@
 using KubernetesWorkflow;
-using Logging;
 using NethereumWorkflow;
 
 namespace DistTestCore.Marketplace
 {
     public class GethBootstrapNodeInfo
     {
-        public GethBootstrapNodeInfo(RunningContainers runningContainers, string account, string genesisJsonBase64, string pubKey, Port discoveryPort)
+        public GethBootstrapNodeInfo(RunningContainers runningContainers, AllGethAccounts allAccounts, string pubKey, Port discoveryPort)
         {
             RunningContainers = runningContainers;
-            Account = account;
-            GenesisJsonBase64 = genesisJsonBase64;
+            AllAccounts = allAccounts;
+            Account = allAccounts.Accounts[0];
             PubKey = pubKey;
             DiscoveryPort = discoveryPort;
         }
 
         public RunningContainers RunningContainers { get; }
-        public string Account { get; }
-        public string GenesisJsonBase64 { get; }
+        public AllGethAccounts AllAccounts { get; }
+        public GethAccount Account { get; }
         public string PubKey { get; }
         public Port DiscoveryPort { get; }
 
-        public NethereumInteraction StartInteraction(TestLog log)
+        public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
         {
-            var ip = RunningContainers.RunningPod.Cluster.IP;
-            var port = RunningContainers.Containers[0].ServicePorts[0].Number;
+            var address = lifecycle.Configuration.GetAddress(RunningContainers.Containers[0]);
+            var account = Account;
 
-            var creator = new NethereumInteractionCreator(log, ip, port, Account);
+            var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, account.PrivateKey);
             return creator.CreateWorkflow();
         }
     }
+
+    public class AllGethAccounts
+    {
+        public GethAccount[] Accounts { get; }
+
+        public AllGethAccounts(GethAccount[] accounts)
+        {
+            Accounts = accounts;
+        }
+    }
 }

DistTestCore/Marketplace/GethBootstrapNodeStarter.cs

@@ -2,48 +2,39 @@
 
 namespace DistTestCore.Marketplace
 {
-    public class GethBootstrapNodeStarter
+    public class GethBootstrapNodeStarter : BaseStarter
     {
-        private const string bootstrapGenesisJsonBase64 = "ewogICAgImNvbmZpZyI6IHsKICAgICAgImNoYWluSWQiOiA3ODk5ODgsCiAgICAgICJob21lc3RlYWRCbG9jayI6IDAsCiAgICAgICJlaXAxNTBCbG9jayI6IDAsCiAgICAgICJlaXAxNTVCbG9jayI6IDAsCiAgICAgICJlaXAxNThCbG9jayI6IDAsCiAgICAgICJieXphbnRpdW1CbG9jayI6IDAsCiAgICAgICJjb25zdGFudGlub3BsZUJsb2NrIjogMCwKICAgICAgInBldGVyc2J1cmdCbG9jayI6IDAsCiAgICAgICJpc3RhbmJ1bEJsb2NrIjogMCwKICAgICAgIm11aXJHbGFjaWVyQmxvY2siOiAwLAogICAgICAiYmVybGluQmxvY2siOiAwLAogICAgICAibG9uZG9uQmxvY2siOiAwLAogICAgICAiYXJyb3dHbGFjaWVyQmxvY2siOiAwLAogICAgICAiZ3JheUdsYWNpZXJCbG9jayI6IDAsCiAgICAgICJjbGlxdWUiOiB7CiAgICAgICAgInBlcmlvZCI6IDUsCiAgICAgICAgImVwb2NoIjogMzAwMDAKICAgICAgfQogICAgfSwKICAgICJkaWZmaWN1bHR5IjogIjEiLAogICAgImdhc0xpbWl0IjogIjgwMDAwMDAwMCIsCiAgICAiZXh0cmFkYXRhIjogIjB4MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMEFDQ09VTlRfSEVSRTAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiLAogICAgImFsbG9jIjogewogICAgICAiMHhBQ0NPVU5UX0hFUkUiOiB7ICJiYWxhbmNlIjogIjUwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAiIH0KICAgIH0KICB9";
-        private readonly TestLifecycle lifecycle;
-        private readonly WorkflowCreator workflowCreator;
-
-        public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+        public GethBootstrapNodeStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
-            this.lifecycle = lifecycle;
-            this.workflowCreator = workflowCreator;
         }
 
         public GethBootstrapNodeInfo StartGethBootstrapNode()
         {
-            Log("Starting Geth bootstrap node...");
+            LogStart("Starting Geth bootstrap node...");
             var startupConfig = CreateBootstrapStartupConfig();
             
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
+            var bootstrapContainer = containers.Containers[0];
 
-            var extractor = new GethInfoExtractor(workflow, containers.Containers[0]);
-            var account = extractor.ExtractAccount();
-            var genesisJsonBase64 = extractor.ExtractGenesisJsonBase64();
+            var extractor = new ContainerInfoExtractor(lifecycle.Log, workflow, bootstrapContainer);
+            var accounts = extractor.ExtractAccounts();
             var pubKey = extractor.ExtractPubKey();
-            var discoveryPort = containers.Containers[0].Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
+            var discoveryPort = bootstrapContainer.Recipe.GetPortByTag(GethContainerRecipe.DiscoveryPortTag);
+            var result = new GethBootstrapNodeInfo(containers, accounts, pubKey, discoveryPort);
 
-            Log($"Geth bootstrap node started with account '{account}'");
+            LogEnd($"Geth bootstrap node started with account '{result.Account.Account}'");
 
-            return new GethBootstrapNodeInfo(containers, account, genesisJsonBase64, pubKey, discoveryPort);
+            return result;
         }
 
         private StartupConfig CreateBootstrapStartupConfig()
         {
             var config = new StartupConfig();
-            config.Add(new GethStartupConfig(true, bootstrapGenesisJsonBase64, null!));
+            config.Add(new GethStartupConfig(true, null!, 0, 0));
             return config;
         }
-
-        private void Log(string msg)
-        {
-            lifecycle.Log.Log(msg);
-        }
     }
 }

DistTestCore/Marketplace/GethCompanionNodeInfo.cs

@@ -1,16 +1,38 @@
 using KubernetesWorkflow;
+using NethereumWorkflow;
 
 namespace DistTestCore.Marketplace
 {
     public class GethCompanionNodeInfo
     {
-        public GethCompanionNodeInfo(RunningContainer runningContainer, string account)
+        public GethCompanionNodeInfo(RunningContainer runningContainer, GethAccount[] accounts)
         {
             RunningContainer = runningContainer;
-            Account = account;
+            Accounts = accounts;
         }
 
         public RunningContainer RunningContainer { get; }
+        public GethAccount[] Accounts { get; }
+        
+        public NethereumInteraction StartInteraction(TestLifecycle lifecycle, GethAccount account)
+        {
+            var address = lifecycle.Configuration.GetAddress(RunningContainer);
+            var privateKey = account.PrivateKey;
+
+            var creator = new NethereumInteractionCreator(lifecycle.Log, address.Host, address.Port, privateKey);
+            return creator.CreateWorkflow();
+        }
+    }
+
+    public class GethAccount
+    {
+        public GethAccount(string account, string privateKey)
+        {
+            Account = account;
+            PrivateKey = privateKey;
+        }
+
         public string Account { get; }
+        public string PrivateKey { get; }
     }
 }

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -1,50 +1,77 @@
 using KubernetesWorkflow;
+using Utils;
 
 namespace DistTestCore.Marketplace
 {
-    public class GethCompanionNodeStarter
+    public class GethCompanionNodeStarter : BaseStarter
     {
-        private readonly TestLifecycle lifecycle;
-        private readonly WorkflowCreator workflowCreator;
+        private int companionAccountIndex = 0;
 
-        public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+        public GethCompanionNodeStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
-            this.lifecycle = lifecycle;
-            this.workflowCreator = workflowCreator;
         }
 
-        public GethCompanionNodeInfo[] StartCompanionNodesFor(CodexSetup codexSetup, GethBootstrapNodeInfo bootstrapNode)
+        public GethCompanionNodeInfo StartCompanionNodeFor(CodexSetup codexSetup, MarketplaceNetwork marketplace)
         {
-            Log($"Initializing companions for {codexSetup.NumberOfNodes} Codex nodes.");
+            LogStart($"Initializing companion for {codexSetup.NumberOfNodes} Codex nodes.");
 
-            var startupConfig = CreateCompanionNodeStartupConfig(bootstrapNode);
+            var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
 
-            var workflow = workflowCreator.CreateWorkflow();
-            var containers = workflow.Start(codexSetup.NumberOfNodes, Location.Unspecified, new GethContainerRecipe(), startupConfig);
-            if (containers.Containers.Length != codexSetup.NumberOfNodes) throw new InvalidOperationException("Expected a Geth companion node to be created for each Codex node. Test infra failure.");
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
+            if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
+            var container = containers.Containers[0];
 
-            Log("Initialized companion nodes.");
+            var node = CreateCompanionInfo(container, marketplace, config);
+            EnsureCompanionNodeIsSynced(node, marketplace);
 
-            return containers.Containers.Select(c => CreateCompanionInfo(workflow, c)).ToArray();
+            LogEnd($"Initialized one companion node for {codexSetup.NumberOfNodes} Codex nodes. Their accounts: [{string.Join(",", node.Accounts.Select(a => a.Account))}]");
+            return node;
         }
 
-        private GethCompanionNodeInfo CreateCompanionInfo(StartupWorkflow workflow, RunningContainer container)
+        private GethCompanionNodeInfo CreateCompanionInfo(RunningContainer container, MarketplaceNetwork marketplace, GethStartupConfig config)
         {
-            var extractor = new GethInfoExtractor(workflow, container);
-            var account = extractor.ExtractAccount();
-            return new GethCompanionNodeInfo(container, account);
+            var accounts = ExtractAccounts(marketplace, config);
+            return new GethCompanionNodeInfo(container, accounts);
         }
 
-        private StartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode)
+        private static GethAccount[] ExtractAccounts(MarketplaceNetwork marketplace, GethStartupConfig config)
         {
-            var config = new StartupConfig();
-            config.Add(new GethStartupConfig(false, bootstrapNode.GenesisJsonBase64, bootstrapNode));
+            return marketplace.Bootstrap.AllAccounts.Accounts
+                .Skip(1 + config.CompanionAccountStartIndex)
+                .Take(config.NumberOfCompanionAccounts)
+                .ToArray();
+        }
+
+        private void EnsureCompanionNodeIsSynced(GethCompanionNodeInfo node, MarketplaceNetwork marketplace)
+        {
+            try
+            {
+                Time.WaitUntil(() =>
+                {
+                    var interaction = node.StartInteraction(lifecycle, node.Accounts.First());
+                    return interaction.IsSynced(marketplace.Marketplace.Address, marketplace.Marketplace.Abi);
+                }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
+            }
+            catch (Exception e)
+            {
+                throw new Exception("Geth companion node did not sync within timeout. Test infra failure.", e);
+            }
+        }
+
+        private GethStartupConfig CreateCompanionNodeStartupConfig(GethBootstrapNodeInfo bootstrapNode, int numberOfAccounts)
+        {
+            var config = new GethStartupConfig(false, bootstrapNode, companionAccountIndex, numberOfAccounts);
+            companionAccountIndex += numberOfAccounts;
             return config;
         }
 
-        private void Log(string msg)
+        private StartupConfig CreateStartupConfig(GethStartupConfig gethConfig)
         {
-            lifecycle.Log.Log(msg);
+            var config = new StartupConfig();
+            config.Add(gethConfig);
+            return config;
         }
     }
 }

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -4,13 +4,14 @@ namespace DistTestCore.Marketplace
 {
     public class GethContainerRecipe : ContainerRecipeFactory
     {
-        public const string DockerImage = "thatbenbierens/geth-confenv:latest";
+        private const string defaultArgs = "--ipcdisable --syncmode full";
+
         public const string HttpPortTag = "http_port";
         public const string DiscoveryPortTag = "disc_port";
-        public const string AccountFilename = "account_string.txt";
-        public const string GenesisFilename = "genesis.json";
-        
-        protected override string Image => DockerImage;
+        public const string AccountsFilename = "accounts.csv";
+
+        public override string AppName => "geth";
+        public override string Image => "codexstorage/dist-tests-geth:latest";
 
         protected override void Initialize(StartupConfig startupConfig)
         {
@@ -19,7 +20,6 @@ namespace DistTestCore.Marketplace
             var args = CreateArgs(config);
 
             AddEnvVar("GETH_ARGS", args);
-            AddEnvVar("GENESIS_JSON", config.GenesisJsonBase64);
         }
 
         private string CreateArgs(GethStartupConfig config)
@@ -28,22 +28,46 @@ namespace DistTestCore.Marketplace
 
             if (config.IsBootstrapNode)
             {
-                AddEnvVar("IS_BOOTSTRAP", "1");
-                var exposedPort = AddExposedPort();
-                return $"--http.port {exposedPort.Number} --discovery.port {discovery.Number} --nodiscover";
+                return CreateBootstapArgs(discovery);
             }
 
+            return CreateCompanionArgs(discovery, config);
+        }
+
+        private string CreateBootstapArgs(Port discovery)
+        {
+            AddEnvVar("ENABLE_MINER", "1");
+            UnlockAccounts(0, 1);
+            var exposedPort = AddExposedPort(tag: HttpPortTag);
+            return $"--http.port {exposedPort.Number} --port {discovery.Number} --discovery.port {discovery.Number} {defaultArgs}";
+        }
+
+        private string CreateCompanionArgs(Port discovery, GethStartupConfig config)
+        {
+            UnlockAccounts(
+                config.CompanionAccountStartIndex + 1,
+                config.NumberOfCompanionAccounts);
+
             var port = AddInternalPort();
             var authRpc = AddInternalPort();
-            var httpPort = AddInternalPort(tag: HttpPortTag);
+            var httpPort = AddExposedPort(tag: HttpPortTag);
 
             var bootPubKey = config.BootstrapNode.PubKey;
-            var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.Ip;
+            var bootIp = config.BootstrapNode.RunningContainers.Containers[0].Pod.PodInfo.Ip;
             var bootPort = config.BootstrapNode.DiscoveryPort.Number;
-            var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort}";
-            // geth --bootnodes enode://pubkey1@ip1:port1
+            var bootstrapArg = $"--bootnodes enode://{bootPubKey}@{bootIp}:{bootPort} --nat=extip:{bootIp}";
 
-            return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.port {httpPort.Number} --nodiscover {bootstrapArg}";
+            return $"--port {port.Number} --discovery.port {discovery.Number} --authrpc.port {authRpc.Number} --http.addr 0.0.0.0 --http.port {httpPort.Number} --ws --ws.addr 0.0.0.0 --ws.port {httpPort.Number} {bootstrapArg} {defaultArgs}";
+        }
+
+        private void UnlockAccounts(int startIndex, int numberOfAccounts)
+        {
+            if (startIndex < 0) throw new ArgumentException();
+            if (numberOfAccounts < 1) throw new ArgumentException();
+            if (startIndex + numberOfAccounts > 1000) throw new ArgumentException("Out of accounts!");
+
+            AddEnvVar("UNLOCK_START_INDEX", startIndex.ToString());
+            AddEnvVar("UNLOCK_NUMBER", numberOfAccounts.ToString());
         }
     }
 }

DistTestCore/Marketplace/GethInfoExtractor.cs

@@ -1,110 +0,0 @@
-using KubernetesWorkflow;
-using System.Text;
-
-namespace DistTestCore.Marketplace
-{
-    public class GethInfoExtractor
-    {
-        private readonly StartupWorkflow workflow;
-        private readonly RunningContainer container;
-
-        public GethInfoExtractor(StartupWorkflow workflow, RunningContainer container)
-        {
-            this.workflow = workflow;
-            this.container = container;
-        }
-
-        public string ExtractAccount()
-        {
-            var account = Retry(FetchAccount);
-            if (string.IsNullOrEmpty(account)) throw new InvalidOperationException("Unable to fetch account for geth node. Test infra failure.");
-
-            return account;
-        }
-
-        public string ExtractGenesisJsonBase64()
-        {
-            var genesisJson = Retry(FetchGenesisJson);
-            if (string.IsNullOrEmpty(genesisJson)) throw new InvalidOperationException("Unable to fetch genesis-json for geth node. Test infra failure.");
-
-            return Convert.ToBase64String(Encoding.ASCII.GetBytes(genesisJson));
-        }
-
-        public string ExtractPubKey()
-        {
-            var pubKey = Retry(FetchPubKey);
-            if (string.IsNullOrEmpty(pubKey)) throw new InvalidOperationException("Unable to fetch enode from geth node. Test infra failure.");
-
-            return pubKey;
-        }
-
-        private string Retry(Func<string> fetch)
-        {
-            var result = Catch(fetch);
-            if (string.IsNullOrEmpty(result))
-            {
-                Thread.Sleep(TimeSpan.FromSeconds(5));
-                result = fetch();
-            }
-            return result;
-        }
-
-        private string Catch(Func<string> fetch)
-        {
-            try
-            {
-                return fetch();
-            }
-            catch
-            {
-                return string.Empty;
-            }
-        }
-
-        private string FetchGenesisJson()
-        {
-            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.GenesisFilename);
-        }
-
-        private string FetchAccount()
-        {
-            return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountFilename);
-        }
-
-        private string FetchPubKey()
-        {
-            var enodeFinder = new PubKeyFinder();
-            workflow.DownloadContainerLog(container, enodeFinder);
-            return enodeFinder.GetPubKey();
-        }
-    }
-
-    public class PubKeyFinder : LogHandler, ILogHandler
-    {
-        private const string openTag = "self=\"enode://";
-        private string pubKey = string.Empty;
-
-        public string GetPubKey()
-        {
-            return pubKey;
-        }
-
-        protected override void ProcessLine(string line)
-        {
-            if (line.Contains(openTag))
-            {
-                ExtractPubKey(line);
-            }
-        }
-
-        private void ExtractPubKey(string line)
-        {
-            var openIndex = line.IndexOf(openTag) + openTag.Length;
-            var closeIndex = line.IndexOf("@");
-
-            pubKey = line.Substring(
-                    startIndex: openIndex,
-                    length: closeIndex - openIndex);
-        }
-    }
-}

DistTestCore/Marketplace/GethStartResult.cs

@@ -1,16 +1,19 @@
-namespace DistTestCore.Marketplace
+using Newtonsoft.Json;
+
+namespace DistTestCore.Marketplace
 {
     public class GethStartResult
     {
-        public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo[] companionNodes)
+        public GethStartResult(IMarketplaceAccessFactory marketplaceAccessFactory, MarketplaceNetwork marketplaceNetwork, GethCompanionNodeInfo companionNode)
         {
             MarketplaceAccessFactory = marketplaceAccessFactory;
-            BootstrapNode = bootstrapNode;
-            CompanionNodes = companionNodes;
+            MarketplaceNetwork = marketplaceNetwork;
+            CompanionNode = companionNode;
         }
 
+        [JsonIgnore]
         public IMarketplaceAccessFactory MarketplaceAccessFactory { get; }
-        public GethBootstrapNodeInfo BootstrapNode { get; }
-        public GethCompanionNodeInfo[] CompanionNodes { get; }
+        public MarketplaceNetwork MarketplaceNetwork { get; }
+        public GethCompanionNodeInfo CompanionNode { get; }
     }
 }

DistTestCore/Marketplace/GethStartupConfig.cs

@@ -2,15 +2,17 @@
 {
     public class GethStartupConfig
     {
-        public GethStartupConfig(bool isBootstrapNode, string genesisJsonBase64, GethBootstrapNodeInfo bootstrapNode)
+        public GethStartupConfig(bool isBootstrapNode, GethBootstrapNodeInfo bootstrapNode, int companionAccountStartIndex, int numberOfCompanionAccounts)
         {
             IsBootstrapNode = isBootstrapNode;
-            GenesisJsonBase64 = genesisJsonBase64;
             BootstrapNode = bootstrapNode;
+            CompanionAccountStartIndex = companionAccountStartIndex;
+            NumberOfCompanionAccounts = numberOfCompanionAccounts;
         }
 
         public bool IsBootstrapNode { get; }
-        public string GenesisJsonBase64 { get; }
         public GethBootstrapNodeInfo BootstrapNode { get; }
+        public int CompanionAccountStartIndex { get; }
+        public int NumberOfCompanionAccounts { get; }
     }
 }

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -1,62 +1,138 @@
-using Logging;
+using DistTestCore.Codex;
+using DistTestCore.Helpers;
+using Logging;
+using Newtonsoft.Json;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
+using System.Numerics;
+using Utils;
 
 namespace DistTestCore.Marketplace
 {
     public interface IMarketplaceAccess
     {
-        void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral);
-        void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes);
+        string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration);
+        StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
         void AssertThatBalance(IResolveConstraint constraint, string message = "");
-        decimal GetBalance();
+        TestToken GetBalance();
     }
 
     public class MarketplaceAccess : IMarketplaceAccess
     {
-        private readonly TestLog log;
-        private readonly GethBootstrapNodeInfo bootstrapNode;
-        private readonly GethCompanionNodeInfo companionNode;
+        private readonly TestLifecycle lifecycle;
+        private readonly MarketplaceNetwork marketplaceNetwork;
+        private readonly GethAccount account;
+        private readonly CodexAccess codexAccess;
 
-        public MarketplaceAccess(TestLog log, GethBootstrapNodeInfo bootstrapNode, GethCompanionNodeInfo companionNode)
+        public MarketplaceAccess(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork, GethAccount account, CodexAccess codexAccess)
         {
-            this.log = log;
-            this.bootstrapNode = bootstrapNode;
-            this.companionNode = companionNode;
+            this.lifecycle = lifecycle;
+            this.marketplaceNetwork = marketplaceNetwork;
+            this.account = account;
+            this.codexAccess = codexAccess;
         }
 
-        public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes)
+        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
         {
-            throw new NotImplementedException();
+            var request = new CodexSalesRequestStorageRequest
+            {
+                duration = ToDecInt(duration.TotalSeconds),
+                proofProbability = ToDecInt(proofProbability),
+                reward = ToDecInt(pricePerSlotPerSecond),
+                collateral = ToDecInt(requiredCollateral),
+                expiry = null,
+                nodes = minRequiredNumberOfNodes,
+                tolerance = null,
+            };
+
+            Log($"Requesting storage for: {contentId.Id}... (" +
+                $"pricePerSlotPerSecond: {pricePerSlotPerSecond}, " +
+                $"requiredCollateral: {requiredCollateral}, " +
+                $"minRequiredNumberOfNodes: {minRequiredNumberOfNodes}, " +
+                $"proofProbability: {proofProbability}, " +
+                $"duration: {Time.FormatDuration(duration)})");
+
+            var response = codexAccess.RequestStorage(request, contentId.Id);
+
+            if (response == "Purchasing not available")
+            {
+                throw new InvalidOperationException(response);
+            }
+
+            Log($"Storage requested successfully. PurchaseId: '{response}'.");
+
+            return new StoragePurchaseContract(lifecycle.Log, codexAccess, response, duration);
         }
 
-        public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral)
+        public string MakeStorageAvailable(ByteSize totalSpace, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration)
         {
-            throw new NotImplementedException();
+            var request = new CodexSalesAvailabilityRequest
+            {
+                size = ToDecInt(totalSpace.SizeInBytes),
+                duration = ToDecInt(maxDuration.TotalSeconds),
+                maxCollateral = ToDecInt(maxCollateral),
+                minPrice = ToDecInt(minPriceForTotalSpace)
+            };
+
+            Log($"Making storage available... (" +
+                $"size: {totalSpace}, " +
+                $"minPriceForTotalSpace: {minPriceForTotalSpace}, " +
+                $"maxCollateral: {maxCollateral}, " +
+                $"maxDuration: {Time.FormatDuration(maxDuration)})");
+
+            var response = codexAccess.SalesAvailability(request);
+
+            Log($"Storage successfully made available. Id: {response.id}");
+
+            return response.id;
+        }
+
+        private string ToDecInt(double d)
+        {
+            var i = new BigInteger(d);
+            return i.ToString("D");
+        }
+
+        public string ToDecInt(TestToken t)
+        {
+            var i = new BigInteger(t.Amount);
+            return i.ToString("D");
         }
 
         public void AssertThatBalance(IResolveConstraint constraint, string message = "")
         {
-            Assert.That(GetBalance(), constraint, message);
+            AssertHelpers.RetryAssert(constraint, GetBalance, message);
         }
 
-        public decimal GetBalance()
+        public TestToken GetBalance()
         {
-            var interaction = bootstrapNode.StartInteraction(log);
-            return interaction.GetBalance(companionNode.Account);
+            var interaction = marketplaceNetwork.StartInteraction(lifecycle);
+            var amount = interaction.GetBalance(marketplaceNetwork.Marketplace.TokenAddress, account.Account);
+            var balance = new TestToken(amount);
+
+            Log($"Balance of {account.Account} is {balance}.");
+
+            return balance;
+        }
+
+        private void Log(string msg)
+        {
+            lifecycle.Log.Log($"{codexAccess.Container.Name} {msg}");
         }
     }
 
     public class MarketplaceUnavailable : IMarketplaceAccess
     {
-        public void RequestStorage(ContentId contentId, int pricePerBytePerSecond, float requiredCollateral, float minRequiredNumberOfNodes)
+        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
         {
             Unavailable();
+            return null!;
         }
 
-        public void MakeStorageAvailable(ByteSize size, int minPricePerBytePerSecond, float maxCollateral)
+        public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration)
         {
             Unavailable();
+            return string.Empty;
         }
 
         public void AssertThatBalance(IResolveConstraint constraint, string message = "")
@@ -64,10 +140,10 @@ namespace DistTestCore.Marketplace
             Unavailable();
         }
 
-        public decimal GetBalance()
+        public TestToken GetBalance()
         {
             Unavailable();
-            return 0;
+            return new TestToken(0);
         }
 
         private void Unavailable()
@@ -76,4 +152,92 @@ namespace DistTestCore.Marketplace
             throw new InvalidOperationException();
         }
     }
+
+    public class StoragePurchaseContract
+    {
+        private readonly BaseLog log;
+        private readonly CodexAccess codexAccess;
+        private DateTime? contractStartUtc;
+
+        public StoragePurchaseContract(BaseLog log, CodexAccess codexAccess, string purchaseId, TimeSpan contractDuration)
+        {
+            this.log = log;
+            this.codexAccess = codexAccess;
+            PurchaseId = purchaseId;
+            ContractDuration = contractDuration;
+        }
+
+        public string PurchaseId { get; }
+        public TimeSpan ContractDuration { get; }
+
+        public void WaitForStorageContractStarted()
+        {
+            WaitForStorageContractStarted(TimeSpan.FromSeconds(30));
+        }
+
+        public void WaitForStorageContractFinished()
+        {
+            if (!contractStartUtc.HasValue)
+            {
+                WaitForStorageContractStarted();
+            }
+            var gracePeriod = TimeSpan.FromSeconds(10);
+            var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
+            var timeout = (ContractDuration - currentContractTime) + gracePeriod;
+            WaitForStorageContractState(timeout, "finished");
+        }
+
+        /// <summary>
+        /// Wait for contract to start. Max timeout depends on contract filesize. Allows more time for larger files.
+        /// </summary>
+        public void WaitForStorageContractStarted(ByteSize contractFileSize)
+        {
+            var filesizeInMb = contractFileSize.SizeInBytes / (1024 * 1024);
+            var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0);
+
+            WaitForStorageContractStarted(maxWaitTime);
+        }
+
+        public void WaitForStorageContractStarted(TimeSpan timeout)
+        {
+            WaitForStorageContractState(timeout, "started");
+            contractStartUtc = DateTime.UtcNow;
+        }
+
+        private void WaitForStorageContractState(TimeSpan timeout, string desiredState)
+        {
+            var lastState = "";
+            var waitStart = DateTime.UtcNow;
+            
+            log.Log($"Waiting for {Time.FormatDuration(timeout)} for contract '{PurchaseId}' to reach state '{desiredState}'.");
+            while (lastState != desiredState)
+            {
+                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
+                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
+                if (purchaseStatus != null && purchaseStatus.state != lastState)
+                {
+                    lastState = purchaseStatus.state;
+                    log.Debug("Purchase status: " + statusJson);
+                }
+
+                Thread.Sleep(1000);
+
+                if (lastState == "errored")
+                {
+                    Assert.Fail("Contract errored: " + statusJson);
+                }
+
+                if (DateTime.UtcNow - waitStart > timeout)
+                {
+                    Assert.Fail($"Contract did not reach '{desiredState}' within timeout. {statusJson}");
+                }
+            }
+            log.Log($"Contract '{desiredState}'.");
+        }
+
+        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
+        {
+            return codexAccess.GetPurchaseStatus(purchaseId);
+        }
+    }
 }

DistTestCore/Marketplace/MarketplaceAccessFactory.cs

@@ -1,5 +1,4 @@
 using DistTestCore.Codex;
-using Logging;
 
 namespace DistTestCore.Marketplace
 {
@@ -18,25 +17,25 @@ namespace DistTestCore.Marketplace
 
     public class GethMarketplaceAccessFactory : IMarketplaceAccessFactory
     {
-        private readonly TestLog log;
-        private readonly GethBootstrapNodeInfo bootstrapNode;
+        private readonly TestLifecycle lifecycle;
+        private readonly MarketplaceNetwork marketplaceNetwork;
 
-        public GethMarketplaceAccessFactory(TestLog log, GethBootstrapNodeInfo bootstrapNode)
+        public GethMarketplaceAccessFactory(TestLifecycle lifecycle, MarketplaceNetwork marketplaceNetwork)
         {
-            this.log = log;
-            this.bootstrapNode = bootstrapNode;
+            this.lifecycle = lifecycle;
+            this.marketplaceNetwork = marketplaceNetwork;
         }
 
         public IMarketplaceAccess CreateMarketplaceAccess(CodexAccess access)
         {
             var companionNode = GetGethCompanionNode(access);
-            return new MarketplaceAccess(log, bootstrapNode, companionNode);
+            return new MarketplaceAccess(lifecycle, marketplaceNetwork, companionNode, access);
         }
 
-        private GethCompanionNodeInfo GetGethCompanionNode(CodexAccess access)
+        private GethAccount GetGethCompanionNode(CodexAccess access)
         {
-            var node = access.Container.Recipe.Additionals.Single(a => a is GethCompanionNodeInfo);
-            return (GethCompanionNodeInfo)node;
+            var account = access.Container.Recipe.Additionals.Single(a => a is GethAccount);
+            return (GethAccount)account;
         }
     }
 }

DistTestCore/Marketplace/MarketplaceInitialConfig.cs

@@ -2,11 +2,16 @@
 {
     public class MarketplaceInitialConfig
     {
-        public MarketplaceInitialConfig(int initialBalance)
+        public MarketplaceInitialConfig(Ether initialEth, TestToken initialTestTokens, bool isValidator)
         {
-            InitialBalance = initialBalance;
+            InitialEth = initialEth;
+            InitialTestTokens = initialTestTokens;
+            IsValidator = isValidator;
         }
 
-        public int InitialBalance { get; }
+        public Ether InitialEth { get; }
+        public TestToken InitialTestTokens { get; }
+        public bool IsValidator { get; }
+        public int? AccountIndexOverride { get; set; }
     }
 }

DistTestCore/Marketplace/MarketplaceNetwork.cs

@@ -0,0 +1,21 @@
+using NethereumWorkflow;
+
+namespace DistTestCore.Marketplace
+{
+    public class MarketplaceNetwork
+    {
+        public MarketplaceNetwork(GethBootstrapNodeInfo bootstrap, MarketplaceInfo marketplace)
+        {
+            Bootstrap = bootstrap;
+            Marketplace = marketplace;
+        }
+
+        public GethBootstrapNodeInfo Bootstrap { get; }
+        public MarketplaceInfo Marketplace { get; }
+
+        public NethereumInteraction StartInteraction(TestLifecycle lifecycle)
+        {
+            return Bootstrap.StartInteraction(lifecycle);
+        }
+    }
+}

DistTestCore/Metrics/GrafanaContainerRecipe.cs

@@ -0,0 +1,25 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Metrics
+{
+    public class GrafanaContainerRecipe : ContainerRecipeFactory
+    {
+        public override string AppName => "grafana";
+        public override string Image => "grafana/grafana-oss:10.0.3";
+
+        public const string DefaultAdminUser = "adminium";
+        public const string DefaultAdminPassword = "passwordium";
+
+        protected override void Initialize(StartupConfig startupConfig)
+        {
+            AddExposedPort(3000);
+
+            AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true");
+            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org.");
+            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor");
+
+            AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser);
+            AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword);
+        }
+    }
+}

DistTestCore/Metrics/MetricsAccess.cs

@@ -1,4 +1,6 @@
-using KubernetesWorkflow;
+using DistTestCore.Helpers;
+using KubernetesWorkflow;
+using Logging;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
 using Utils;
@@ -10,31 +12,31 @@ namespace DistTestCore.Metrics
         void AssertThat(string metricName, IResolveConstraint constraint, string message = "");
     }
 
-    public class MetricsUnavailable : IMetricsAccess
-    {
-        public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
-        {
-            Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
-            throw new InvalidOperationException();
-        }
-    }
-
     public class MetricsAccess : IMetricsAccess
     {
+        private readonly BaseLog log;
+        private readonly ITimeSet timeSet;
         private readonly MetricsQuery query;
         private readonly RunningContainer node;
 
-        public MetricsAccess(MetricsQuery query, RunningContainer node)
+        public MetricsAccess(BaseLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
         {
+            this.log = log;
+            this.timeSet = timeSet;
             this.query = query;
             this.node = node;
         }
 
         public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
         {
-            var metricSet = GetMetricWithTimeout(metricName);
-            var metricValue = metricSet.Values[0].Value;
-            Assert.That(metricValue, constraint, message);
+            AssertHelpers.RetryAssert(constraint, () =>
+            {
+                var metricSet = GetMetricWithTimeout(metricName);
+                var metricValue = metricSet.Values[0].Value;
+
+                log.Log($"{node.Name} metric '{metricName}' = {metricValue}");
+                return metricValue;
+            }, message);
         }
 
         public Metrics? GetAllMetrics()
@@ -50,7 +52,7 @@ namespace DistTestCore.Metrics
             {
                 var mostRecent = GetMostRecent(metricName);
                 if (mostRecent != null) return mostRecent;
-                if (DateTime.UtcNow - start > Timing.WaitForMetricTimeout())
+                if (DateTime.UtcNow - start > timeSet.WaitForMetricTimeout())
                 {
                     Assert.Fail($"Timeout: Unable to get metric '{metricName}'.");
                     throw new TimeoutException();
@@ -67,4 +69,13 @@ namespace DistTestCore.Metrics
             return result.Sets.LastOrDefault();
         }
     }
+
+    public class MetricsUnavailable : IMetricsAccess
+    {
+        public void AssertThat(string metricName, IResolveConstraint constraint, string message = "")
+        {
+            Assert.Fail("Incorrect test setup: Metrics were not enabled for this group of Codex nodes. Add 'EnableMetrics()' after 'SetupCodexNodes()' to enable it.");
+            throw new InvalidOperationException();
+        }
+    }
 }

DistTestCore/Metrics/MetricsAccessFactory.cs

@@ -17,17 +17,19 @@ namespace DistTestCore.Metrics
 
     public class CodexNodeMetricsAccessFactory : IMetricsAccessFactory
     {
+        private readonly TestLifecycle lifecycle;
         private readonly RunningContainers prometheusContainer;
 
-        public CodexNodeMetricsAccessFactory(RunningContainers prometheusContainer)
+        public CodexNodeMetricsAccessFactory(TestLifecycle lifecycle, RunningContainers prometheusContainer)
         {
+            this.lifecycle = lifecycle;
             this.prometheusContainer = prometheusContainer;
         }
 
         public IMetricsAccess CreateMetricsAccess(RunningContainer codexContainer)
         {
-            var query = new MetricsQuery(prometheusContainer);
-            return new MetricsAccess(query, codexContainer);
+            var query = new MetricsQuery(lifecycle, prometheusContainer);
+            return new MetricsAccess(lifecycle.Log, lifecycle.TimeSet, query, codexContainer);
         }
     }
 }

DistTestCore/Metrics/MetricsDownloader.cs

@@ -5,9 +5,9 @@ namespace DistTestCore.Metrics
 {
     public class MetricsDownloader
     {
-        private readonly TestLog log;
+        private readonly BaseLog log;
 
-        public MetricsDownloader(TestLog log)
+        public MetricsDownloader(BaseLog log)
         {
             this.log = log;
         }

DistTestCore/Metrics/MetricsMode.cs

@@ -0,0 +1,9 @@
+namespace DistTestCore.Metrics
+{
+    public enum MetricsMode
+    {
+        None,
+        Record,
+        Dashboard
+    }
+}

DistTestCore/Metrics/MetricsQuery.cs

@@ -8,13 +8,16 @@ namespace DistTestCore.Metrics
     {
         private readonly Http http;
 
-        public MetricsQuery(RunningContainers runningContainers)
+        public MetricsQuery(TestLifecycle lifecycle, RunningContainers runningContainers)
         {
             RunningContainers = runningContainers;
 
+            var address = lifecycle.Configuration.GetAddress(runningContainers.Containers[0]);
+
             http = new Http(
-                runningContainers.RunningPod.Cluster.IP,
-                runningContainers.Containers[0].ServicePorts[0].Number,
+                lifecycle.Log,
+                lifecycle.TimeSet,
+                address,
                 "api/v1");
         }
 
@@ -116,7 +119,7 @@ namespace DistTestCore.Metrics
 
         private string GetInstanceNameForNode(RunningContainer node)
         {
-            var ip = node.Pod.Ip;
+            var ip = node.Pod.PodInfo.Ip;
             var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
             return $"{ip}:{port}";
         }

DistTestCore/Metrics/PrometheusContainerRecipe.cs

@@ -4,9 +4,8 @@ namespace DistTestCore.Metrics
 {
     public class PrometheusContainerRecipe : ContainerRecipeFactory
     {
-        public const string DockerImage = "thatbenbierens/prometheus-envconf:latest";
-
-        protected override string Image => DockerImage;
+        public override string AppName => "prometheus";
+        public override string Image => "codexstorage/dist-tests-prometheus:latest";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

DistTestCore/OnlineCodexNode.cs

@@ -2,19 +2,25 @@
 using DistTestCore.Logs;
 using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
+using Logging;
 using NUnit.Framework;
+using Utils;
 
 namespace DistTestCore
 {
     public interface IOnlineCodexNode
     {
+        string GetName();
         CodexDebugResponse GetDebugInfo();
+        CodexDebugPeerResponse GetDebugPeer(string peerId);
         ContentId UploadFile(TestFile file);
-        TestFile? DownloadContent(ContentId contentId);
+        TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(IOnlineCodexNode node);
-        ICodexNodeLog DownloadLog();
+        IDownloadedLog DownloadLog(int? tailLines = null);
         IMetricsAccess Metrics { get; }
         IMarketplaceAccess Marketplace { get; }
+        CodexDebugVersionResponse Version { get; }
+        ICodexSetup BringOffline();
     }
 
     public class OnlineCodexNode : IOnlineCodexNode
@@ -30,44 +36,58 @@ namespace DistTestCore
             Group = group;
             Metrics = metricsAccess;
             Marketplace = marketplaceAccess;
+            Version = new CodexDebugVersionResponse();
         }
 
         public CodexAccess CodexAccess { get; }
         public CodexNodeGroup Group { get; }
         public IMetricsAccess Metrics { get; }
         public IMarketplaceAccess Marketplace { get; }
+        public CodexDebugVersionResponse Version { get; private set; }
 
         public string GetName()
         {
-            return $"<{CodexAccess.Container.Recipe.Name}>";
+            return CodexAccess.Container.Name;
         }
 
         public CodexDebugResponse GetDebugInfo()
         {
-            var response = CodexAccess.GetDebugInfo();
-            Log($"Got DebugInfo with id: '{response.id}'.");
-            return response;
+            var debugInfo = CodexAccess.GetDebugInfo();
+            var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
+            Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
+            return debugInfo;
+        }
+
+        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        {
+            return CodexAccess.GetDebugPeer(peerId);
         }
 
         public ContentId UploadFile(TestFile file)
         {
-            Log($"Uploading file of size {file.GetFileSize()}...");
             using var fileStream = File.OpenRead(file.Filename);
-            var response = CodexAccess.UploadFile(fileStream);
-            if (response.StartsWith(UploadFailedMessage))
+
+            var logMessage = $"Uploading file {file.Describe()}...";
+            Log(logMessage);
+            var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
             {
-                Assert.Fail("Node failed to store block.");
-            }
+                return CodexAccess.UploadFile(fileStream);
+            });
+
+            if (string.IsNullOrEmpty(response)) Assert.Fail("Received empty response.");
+            if (response.StartsWith(UploadFailedMessage)) Assert.Fail("Node failed to store block.");
+
             Log($"Uploaded file. Received contentId: '{response}'.");
             return new ContentId(response);
         }
 
-        public TestFile? DownloadContent(ContentId contentId)
+        public TestFile? DownloadContent(ContentId contentId, string fileLabel = "")
         {
-            Log($"Downloading for contentId: '{contentId.Id}'...");
-            var file = lifecycle.FileManager.CreateEmptyTestFile();
-            DownloadToFile(contentId.Id, file);
-            Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'.");
+            var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
+            Log(logMessage);
+            var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel);
+            Stopwatch.Measure(lifecycle.Log, logMessage, () => DownloadToFile(contentId.Id, file));
+            Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
             return file;
         }
 
@@ -83,14 +103,34 @@ namespace DistTestCore
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public ICodexNodeLog DownloadLog()
+        public IDownloadedLog DownloadLog(int? tailLines = null)
         {
-            return lifecycle.DownloadLog(this);
+            return lifecycle.DownloadLog(CodexAccess.Container, tailLines);
         }
 
-        public string Describe()
+        public ICodexSetup BringOffline()
         {
-            return $"({Group.Describe()} contains {GetName()})";
+            if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
+                "individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
+                "available for codex-nodes in groups of 1.");
+
+            return Group.BringOffline();
+        }
+
+        public void EnsureOnlineGetVersionResponse()
+        {
+            var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
+            var nodePeerId = debugInfo.id;
+            var nodeName = CodexAccess.Container.Name;
+
+            if (!debugInfo.codex.IsValid())
+            {
+                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}");
+            }
+
+            lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
+            lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
+            Version = debugInfo.codex;
         }
 
         private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
@@ -98,21 +138,24 @@ namespace DistTestCore
             var multiAddress = peerInfo.addrs.First();
             // Todo: Is there a case where First address in list is not the way?
 
-            if (Group == peer.Group)
-            {
-                return multiAddress;
-            }
-
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.Ip);
+            return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip);
         }
 
         private void DownloadToFile(string contentId, TestFile file)
         {
             using var fileStream = File.OpenWrite(file.Filename);
-            using var downloadStream = CodexAccess.DownloadFile(contentId);
-            downloadStream.CopyTo(fileStream);
+            try
+            {
+                using var downloadStream = CodexAccess.DownloadFile(contentId);
+                downloadStream.CopyTo(fileStream);
+            }
+            catch
+            {
+                Log($"Failed to download file '{contentId}'.");
+                throw;
+            }
         }
 
         private void Log(string msg)

DistTestCore/PrometheusStarter.cs

@@ -5,39 +5,31 @@ using System.Text;
 
 namespace DistTestCore
 {
-    public class PrometheusStarter
+    public class PrometheusStarter : BaseStarter
     {
-        private readonly TestLifecycle lifecycle;
-        private readonly WorkflowCreator workflowCreator;
-
-        public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+        public PrometheusStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
-            this.lifecycle = lifecycle;
-            this.workflowCreator = workflowCreator;
         }
 
-        public IMetricsAccessFactory CollectMetricsFor(CodexSetup codexSetup, RunningContainers containers)
+        public RunningContainers CollectMetricsFor(RunningContainers[] containers)
         {
-            if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
-
-            Log($"Starting metrics server for {containers.Describe()}");
+            LogStart($"Starting metrics server for {containers.Describe()}");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers)));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers())));
 
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 
-            Log("Metrics server started.");
-
-            return new CodexNodeMetricsAccessFactory(runningContainers);
+            return runningContainers;
         }
 
         private string GeneratePrometheusConfig(RunningContainer[] nodes)
         {
             var config = "";
             config += "global:\n";
-            config += "  scrape_interval: 30s\n";
+            config += "  scrape_interval: 10s\n";
             config += "  scrape_timeout: 10s\n";
             config += "\n";
             config += "scrape_configs:\n";
@@ -48,7 +40,7 @@ namespace DistTestCore
 
             foreach (var node in nodes)
             {
-                var ip = node.Pod.Ip;
+                var ip = node.Pod.PodInfo.Ip;
                 var port = node.Recipe.GetPortByTag(CodexContainerRecipe.MetricsPortTag).Number;
                 config += $"          - '{ip}:{port}'\n";
             }
@@ -56,10 +48,5 @@ namespace DistTestCore
             var bytes = Encoding.ASCII.GetBytes(config);
             return Convert.ToBase64String(bytes);
         }
-
-        private void Log(string msg)
-        {
-            lifecycle.Log.Log(msg);
-        }
     }
 }

DistTestCore/Stopwatch.cs

@@ -1,19 +0,0 @@
-using Logging;
-using Utils;
-
-namespace DistTestCore
-{
-    public class Stopwatch
-    {
-        public static void Measure(BaseLog log, string name, Action action)
-        {
-            var start = DateTime.UtcNow;
-
-            action();
-
-            var duration = DateTime.UtcNow - start;
-
-            log.Log($"{name} ({Time.FormatDuration(duration)})");
-        }
-    }
-}

DistTestCore/TestLifecycle.cs

@@ -1,29 +1,47 @@
-using DistTestCore.Logs;
+using DistTestCore.Codex;
+using DistTestCore.Logs;
+using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
 using KubernetesWorkflow;
 using Logging;
+using Utils;
 
 namespace DistTestCore
 {
     public class TestLifecycle
     {
-        private readonly WorkflowCreator workflowCreator;
+        private readonly DateTime testStart;
 
-        public TestLifecycle(TestLog log, Configuration configuration)
+        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testsType, string testNamespace)
         {
             Log = log;
-            workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration());
+            Configuration = configuration;
+            TimeSet = timeSet;
+
+            var podLabels = new PodLabels(testsType, GetApplicationIds());
+            WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), podLabels, testNamespace);
 
             FileManager = new FileManager(Log, configuration);
-            CodexStarter = new CodexStarter(this, workflowCreator);
-            PrometheusStarter = new PrometheusStarter(this, workflowCreator);
-            GethStarter = new GethStarter(this, workflowCreator);
+            CodexStarter = new CodexStarter(this);
+            PrometheusStarter = new PrometheusStarter(this);
+            GrafanaStarter = new GrafanaStarter(this);
+            GethStarter = new GethStarter(this);
+            testStart = DateTime.UtcNow;
+            CodexVersion = null;
+
+            Log.WriteLogTag();
         }
 
-        public TestLog Log { get; }
+        public BaseLog Log { get; }
+        public Configuration Configuration { get; }
+        public ITimeSet TimeSet { get; }
+        public WorkflowCreator WorkflowCreator { get; }
         public FileManager FileManager { get; }
         public CodexStarter CodexStarter { get; }
         public PrometheusStarter PrometheusStarter { get; }
+        public GrafanaStarter GrafanaStarter { get; }
         public GethStarter GethStarter { get; }
+        public CodexDebugVersionResponse? CodexVersion { get; private set; }
 
         public void DeleteAllResources()
         {
@@ -31,16 +49,46 @@ namespace DistTestCore
             FileManager.DeleteAllTestFiles();
         }
 
-        public ICodexNodeLog DownloadLog(OnlineCodexNode node)
+        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
         {
             var subFile = Log.CreateSubfile();
-            var description = node.Describe();
-            var handler = new LogDownloadHandler(description, subFile);
+            var description = container.Name;
+            var handler = new LogDownloadHandler(container, description, subFile);
 
             Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-            CodexStarter.DownloadLog(node.CodexAccess.Container, handler);
+            CodexStarter.DownloadLog(container, handler, tailLines);
 
-            return new CodexNodeLog(subFile);
+            return new DownloadedLog(subFile, description);
+        }
+
+        public string GetTestDuration()
+        {
+            var testDuration = DateTime.UtcNow - testStart;
+            return Time.FormatDuration(testDuration);
+        }
+
+        public void SetCodexVersion(CodexDebugVersionResponse version)
+        {
+            if (CodexVersion == null) CodexVersion = version;
+        }
+
+        public ApplicationIds GetApplicationIds()
+        {
+            return new ApplicationIds(
+                codexId: GetCodexId(),
+                gethId: new GethContainerRecipe().Image,
+                prometheusId: new PrometheusContainerRecipe().Image,
+                codexContractsId: new CodexContractsContainerRecipe().Image,
+                grafanaId: new GrafanaContainerRecipe().Image
+            );
+        }
+
+        private string GetCodexId()
+        {
+            var v = CodexVersion;
+            if (v == null) return new CodexContainerRecipe().Image;
+            if (v.version != "untagged build") return v.version;
+            return v.revision;
         }
     }
 }

DistTestCore/Timing.cs

@@ -1,63 +1,16 @@
 using NUnit.Framework;
-using Utils;
 
 namespace DistTestCore
 {
     [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
     public class UseLongTimeoutsAttribute : PropertyAttribute
     {
-        public UseLongTimeoutsAttribute()
-            : base(Timing.UseLongTimeoutsKey)
-        {
-        }
-    }
-
-    public static class Timing
-    {
-        public const string UseLongTimeoutsKey = "UseLongTimeouts";
-
-        public static TimeSpan HttpCallTimeout()
-        {
-            return GetTimes().HttpCallTimeout();
-        }
-
-        public static int HttpCallRetryCount()
-        {
-            return GetTimes().HttpCallRetryCount();
-        }
-
-        public static void HttpCallRetryDelay()
-        {
-            Time.Sleep(GetTimes().HttpCallRetryDelay());
-        }
-
-        public static TimeSpan K8sServiceDelay()
-        {
-            return GetTimes().WaitForK8sServiceDelay();
-        }
-
-        public static TimeSpan K8sOperationTimeout()
-        {
-            return GetTimes().K8sOperationTimeout();
-        }
-
-        public static TimeSpan WaitForMetricTimeout()
-        {
-            return GetTimes().WaitForMetricTimeout();
-        }
-
-        private static ITimeSet GetTimes()
-        {
-            var testProperties = TestContext.CurrentContext.Test.Properties;
-            if (testProperties.ContainsKey(UseLongTimeoutsKey)) return new LongTimeSet();
-            return new DefaultTimeSet();
-        }
     }
 
     public interface ITimeSet
     {
         TimeSpan HttpCallTimeout();
-        int HttpCallRetryCount();
+        TimeSpan HttpCallRetryTime();
         TimeSpan HttpCallRetryDelay();
         TimeSpan WaitForK8sServiceDelay();
         TimeSpan K8sOperationTimeout();
@@ -68,27 +21,27 @@ namespace DistTestCore
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromSeconds(10);
+            return TimeSpan.FromMinutes(5);
         }
 
-        public int HttpCallRetryCount()
+        public TimeSpan HttpCallRetryTime()
         {
-            return 5;
+            return TimeSpan.FromMinutes(1);
         }
 
         public TimeSpan HttpCallRetryDelay()
         {
-            return TimeSpan.FromSeconds(3);
+            return TimeSpan.FromSeconds(1);
         }
 
         public TimeSpan WaitForK8sServiceDelay()
         {
-            return TimeSpan.FromSeconds(1);
+            return TimeSpan.FromSeconds(10);
         }
 
         public TimeSpan K8sOperationTimeout()
         {
-            return TimeSpan.FromMinutes(5);
+            return TimeSpan.FromMinutes(30);
         }
 
         public TimeSpan WaitForMetricTimeout()
@@ -104,14 +57,14 @@ namespace DistTestCore
             return TimeSpan.FromHours(2);
         }
 
-        public int HttpCallRetryCount()
+        public TimeSpan HttpCallRetryTime()
         {
-            return 2;
+            return TimeSpan.FromHours(5);
         }
 
         public TimeSpan HttpCallRetryDelay()
         {
-            return TimeSpan.FromMinutes(5);
+            return TimeSpan.FromSeconds(2);
         }
 
         public TimeSpan WaitForK8sServiceDelay()

DistTestCore/Tokens.cs

@@ -0,0 +1,97 @@
+namespace DistTestCore
+{
+    public class Ether : IComparable<Ether>
+    {
+        public Ether(decimal wei)
+        {
+            Wei = wei;
+        }
+
+        public decimal Wei { get; }
+
+        public int CompareTo(Ether? other)
+        {
+            return Wei.CompareTo(other!.Wei);
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is Ether ether && Wei == ether.Wei;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Wei);
+        }
+
+        public override string ToString()
+        {
+            return $"{Wei} Wei";
+        }
+    }
+
+    public class TestToken : IComparable<TestToken>
+    {
+        public TestToken(decimal amount)
+        {
+            Amount = amount;
+        }
+
+        public decimal Amount { get; }
+
+        public int CompareTo(TestToken? other)
+        {
+            return Amount.CompareTo(other!.Amount);
+        }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is TestToken token && Amount == token.Amount;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Amount);
+        }
+
+        public override string ToString()
+        {
+            return $"{Amount} TestTokens";
+        }
+    }
+
+    public static class TokensIntExtensions
+    {
+        private const decimal weiPerEth = 1000000000000000000;
+
+        public static TestToken TestTokens(this int i)
+        {
+            return TestTokens(Convert.ToDecimal(i));
+        }
+
+        public static TestToken TestTokens(this decimal i)
+        {
+            return new TestToken(i);
+        }
+
+        public static Ether Eth(this int i)
+        {
+            return Eth(Convert.ToDecimal(i));
+        }
+
+        public static Ether Wei(this int i)
+        {
+            return Wei(Convert.ToDecimal(i));
+        }
+
+        public static Ether Eth(this decimal i)
+        {
+            return new Ether(i * weiPerEth);
+        }
+
+        public static Ether Wei(this decimal i)
+        {
+            return new Ether(i);
+        }
+    }
+}

KubernetesWorkflow/CommandRunner.cs

@@ -5,7 +5,7 @@ namespace KubernetesWorkflow
 {
     public class CommandRunner
     {
-        private readonly Kubernetes client;
+        private readonly K8sClient client;
         private readonly string k8sNamespace;
         private readonly RunningPod pod;
         private readonly string containerName;
@@ -13,7 +13,7 @@ namespace KubernetesWorkflow
         private readonly string[] arguments;
         private readonly List<string> lines = new List<string>();
 
-        public CommandRunner(Kubernetes client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
+        public CommandRunner(K8sClient client, string k8sNamespace, RunningPod pod, string containerName, string command, string[] arguments)
         {
             this.client = client;
             this.k8sNamespace = k8sNamespace;
@@ -27,8 +27,8 @@ namespace KubernetesWorkflow
         {
             var input = new[] { command }.Concat(arguments).ToArray();
 
-            Time.Wait(client.NamespacedPodExecAsync(
-                pod.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken()));
+            Time.Wait(client.Run(c => c.NamespacedPodExecAsync(
+                pod.PodInfo.Name, k8sNamespace, containerName, input, false, Callback, new CancellationToken())));
         }
 
         public string GetStdOut()

KubernetesWorkflow/Configuration.cs

@@ -2,31 +2,17 @@
 {
     public class Configuration
     {
-        public Configuration(string k8sNamespace, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay, ConfigurationLocationEntry[] locationMap)
+        public Configuration(string k8sNamespacePrefix, string? kubeConfigFile, TimeSpan operationTimeout, TimeSpan retryDelay)
         {
-            K8sNamespace = k8sNamespace;
+            K8sNamespacePrefix = k8sNamespacePrefix;
             KubeConfigFile = kubeConfigFile;
             OperationTimeout = operationTimeout;
             RetryDelay = retryDelay;
-            LocationMap = locationMap;
         }
 
-        public string K8sNamespace { get; }
+        public string K8sNamespacePrefix { get; }
         public string? KubeConfigFile { get; }
         public TimeSpan OperationTimeout { get; }
         public TimeSpan RetryDelay { get; }
-        public ConfigurationLocationEntry[] LocationMap { get; }
-    }
-
-    public class ConfigurationLocationEntry
-    {
-        public ConfigurationLocationEntry(Location location, string workerName)
-        {
-            Location = location;
-            WorkerName = workerName;
-        }
-
-        public Location Location { get; }
-        public string WorkerName { get; }
     }
 }

KubernetesWorkflow/ContainerRecipe.cs

@@ -24,6 +24,14 @@
         {
             return ExposedPorts.Concat(InternalPorts).Single(p => p.Tag == tag);
         }
+
+        public override string ToString()
+        {
+            return $"(container-recipe: {Name}, image: {Image}, " +
+                $"exposedPorts: {string.Join(",", ExposedPorts.Select(p => p.Number))}, " +
+                $"internalPorts: {string.Join(",", InternalPorts.Select(p => p.Number))}, " +
+                $"envVars: {string.Join(",", EnvVars.Select(v => v.Name + ":" + v.Value))}, ";
+        }
     }
 
     public class Port