Sets up continuous test for fetch connectivity.

* Add Prometheus annotations for Codex Pods (#48)

* Remove unnecessary condition (#48)

* Update Annotations to be handled via ContainerRecipe (#48)

* Wires up metrics port to codex pod annotation

* Cleans up handling of pod labels

* Fix annotations names (#48)

---------

Co-authored-by: benbierens <thatbenbierens@gmail.com>

.github/workflows/docker-reusable.yml

@@ -0,0 +1,177 @@
+name: Docker - Reusable
+
+
+on:
+  workflow_call:
+    inputs:
+      docker_file:
+        default: docker/Dockerfile
+        description: Dockerfile
+        required: false
+        type: string
+      docker_repo:
+        default: codexstorage/cs-codex-dist-tests
+        description: DockerHub repository
+        required: false
+        type: string
+      tag_latest:
+        default: true
+        description: Set latest tag for Docker images
+        required: false
+        type: boolean
+      tag_sha:
+        default: true
+        description: Set Git short commit as Docker tag
+        required: false
+        type: boolean
+      tag_suffix:
+        default: ''
+        description: Suffix for Docker images tag
+        required: false
+        type: string
+
+
+env:
+  DOCKER_FILE: ${{ inputs.docker_file }}
+  DOCKER_REPO: ${{ inputs.docker_repo }}
+  TAG_LATEST: ${{ inputs.tag_latest }}
+  TAG_SHA: ${{ inputs.tag_sha }}
+  TAG_SUFFIX: ${{ inputs.tag_suffix }}
+
+
+jobs:
+  # Build platform specific image
+  build:
+    strategy:
+      fail-fast: true
+      matrix:
+        target:
+          - os: linux
+            arch: amd64
+          - os: linux
+            arch: arm64
+        include:
+          - target:
+              os: linux
+              arch: amd64
+            builder: ubuntu-22.04
+          - target:
+              os: linux
+              arch: arm64
+            builder: buildjet-4vcpu-ubuntu-2204-arm
+
+    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
+    runs-on: ${{ matrix.builder }}
+    env:
+      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_REPO }}
+
+      - name: Docker - Set up Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Build and Push by digest
+        id: build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ${{ env.DOCKER_FILE }}
+          platforms: ${{ env.PLATFORM }}
+          push: true
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Docker - Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Docker - Upload digest
+        uses: actions/upload-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+
+  # Publish multi-platform image
+  publish:
+    name: Publish multi-platform image
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Docker - Variables
+        run: |
+          # Adjust custom suffix when set and
+          if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
+            echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
+          fi
+          # Disable SHA tags on tagged release
+          if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
+            echo "TAG_SHA=false" >>$GITHUB_ENV
+          fi
+          # Handle latest and latest-custom using raw
+          if [[ ${{ env.TAG_SHA }} == "false" ]]; then
+            echo "TAG_LATEST=false" >>$GITHUB_ENV
+            echo "TAG_RAW=true" >>$GITHUB_ENV
+            if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
+              echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
+            else
+              echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
+            fi
+          else
+            echo "TAG_RAW=false" >>$GITHUB_ENV
+          fi
+
+      - name: Docker - Download digests
+        uses: actions/download-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests
+
+      - name: Docker - Set up Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_REPO }}
+          flavor: |
+            latest=${{ env.TAG_LATEST }}
+            suffix=${{ env.TAG_SUFFIX }},onlatest=true
+          tags: |
+            type=semver,pattern={{version}}
+            type=raw,enable=${{ env.TAG_RAW }},value=latest
+            type=sha,enable=${{ env.TAG_SHA }}
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
+
+      - name: Docker - Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}

.github/workflows/docker.yml

@@ -11,113 +11,12 @@ on:
       - docker/Dockerfile
       - docker/docker-entrypoint.sh
       - .github/workflows/docker.yml
+      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
 
-env:
-  DOCKER_FILE: docker/Dockerfile
-  DOCKER_REPO: codexstorage/cs-codex-dist-tests
-
-
 jobs:
-  # Build platform specific image
-  build:
-    strategy:
-      fail-fast: true
-      matrix:
-        target:
-          - os: linux
-            arch: amd64
-          - os: linux
-            arch: arm64
-        include:
-          - target:
-              os: linux
-              arch: amd64
-            builder: ubuntu-22.04
-          - target:
-              os: linux
-              arch: arm64
-            builder: buildjet-4vcpu-ubuntu-2204-arm
-
-    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
-    runs-on: ${{ matrix.builder }}
-    outputs:
-      tags-linux-amd64: ${{ steps.tags.outputs.tags-linux-amd64 }}
-      tags-linux-arm64: ${{ steps.tags.outputs.tags-linux-arm64 }}
-    env:
-      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
-      SUFFIX: ${{ format('{0}-{1}', 'linux', matrix.target.arch) }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.DOCKER_REPO }}
-          flavor: |
-            latest=false
-          tags: |
-            type=semver,pattern={{version}},suffix=-${{ env.SUFFIX }}
-            type=sha,suffix=-${{ env.SUFFIX }},enable=${{ !startsWith(github.ref, 'refs/tags/') }}
-
-      - name: Docker - Set tags output
-        id: tags
-        run: |
-          if [[ '${{ matrix.target.os }}' == 'linux' && '${{ matrix.target.arch }}' == 'amd64' ]]; then
-            echo "tags-linux-amd64=${{ steps.meta.outputs.tags }}" >> "$GITHUB_OUTPUT"
-          elif [[ '${{ matrix.target.os }}' == 'linux' && '${{ matrix.target.arch }}' == 'arm64' ]]; then
-            echo "tags-linux-arm64=${{ steps.meta.outputs.tags }}" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Build and Push
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ${{ env.DOCKER_FILE }}
-          platforms: ${{ env.PLATFORM }}
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-
-  # Publish single image
-  publish:
-    name: Push single image
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.DOCKER_REPO }}
-          tags: |
-            type=semver,pattern={{version}}
-            type=sha,enable=${{ !startsWith(github.ref, 'refs/tags/') }}
-
-      - name: Docker - Set tags
-        run: |
-          # Transform multi-line tags in to the comma-seperated
-          TAGS=$(echo "${{ steps.meta.outputs.tags }}" | tr '\n' ',' | awk '{gsub(/,$/,"");}1')
-          echo "TAGS=${TAGS}" >>$GITHUB_ENV
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Create and push manifest images
-        uses: Noelware/docker-manifest-action@master
-        with:
-          inputs: ${{ env.TAGS }}
-          images: ${{ needs.build.outputs.tags-linux-amd64 }},${{ needs.build.outputs.tags-linux-arm64 }}
-          push: true
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    secrets: inherit

CodexNetDeployer/CodexNodeStarter.cs

@@ -8,28 +8,27 @@ namespace CodexNetDeployer
     public class CodexNodeStarter
     {
         private readonly Configuration config;
-        private readonly WorkflowCreator workflowCreator;
         private readonly TestLifecycle lifecycle;
         private readonly GethStartResult gethResult;
         private string bootstrapSpr = "";
         private int validatorsLeft;
 
-        public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
+        public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
         {
             this.config = config;
-            this.workflowCreator = workflowCreator;
             this.lifecycle = lifecycle;
             this.gethResult = gethResult;
             validatorsLeft = numberOfValidators;
         }
 
-        public RunningContainer? Start(int i)
+        public CodexNodeStartResult? Start(int i)
         {
             Console.Write($" - {i} = ");
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var workflowStartup = new StartupConfig();
             workflowStartup.Add(gethResult);
             workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));
+            workflowStartup.NameOverride = GetCodexContainerName(i);
 
             var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
 
@@ -62,7 +61,7 @@ namespace CodexNetDeployer
 
                         if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
                         validatorsLeft--;
-                        return container;
+                        return new CodexNodeStartResult(workflow, container, codexAccess);
                     }
                 }
             }
@@ -77,6 +76,12 @@ namespace CodexNetDeployer
             return null;
         }
 
+        private string GetCodexContainerName(int i)
+        {
+            if (i == 0) return "BOOTSTRAP";
+            return "CODEX" + i;
+        }
+
         private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft)
         {
             var codexStart = new CodexStartupConfig(config.CodexLogLevel);
@@ -86,14 +91,36 @@ namespace CodexNetDeployer
             var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
             marketplaceConfig.AccountIndexOverride = i;
             codexStart.MarketplaceConfig = marketplaceConfig;
-            codexStart.MetricsEnabled = config.RecordMetrics;
+            codexStart.MetricsMode = config.Metrics;
 
             if (config.BlockTTL != Configuration.SecondsIn1Day)
             {
                 codexStart.BlockTTL = config.BlockTTL;
             }
+            if (config.BlockMI != Configuration.TenMinutes)
+            {
+                codexStart.BlockMaintenanceInterval = TimeSpan.FromSeconds(config.BlockMI);
+            }
+            if (config.BlockMN != 1000)
+            {
+                codexStart.BlockMaintenanceNumber = config.BlockMN;
+            }
 
             return codexStart;
         }
     }
+
+    public class CodexNodeStartResult
+    {
+        public CodexNodeStartResult(StartupWorkflow workflow, RunningContainer container, CodexAccess access)
+        {
+            Workflow = workflow;
+            Container = container;
+            Access = access;
+        }
+
+        public StartupWorkflow Workflow { get; }
+        public RunningContainer Container { get; }
+        public CodexAccess Access { get; }
+    }
 }

CodexNetDeployer/Configuration.cs

@@ -1,12 +1,13 @@
 using ArgsUniform;
-using DistTestCore;
 using DistTestCore.Codex;
+using DistTestCore.Metrics;
 
 namespace CodexNetDeployer
 {
     public class Configuration
     {
         public const int SecondsIn1Day = 24 * 60 * 60;
+        public const int TenMinutes = 10 * 60;
 
         [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
         public string KubeConfigFile { get; set; } = "null";
@@ -44,11 +45,22 @@ namespace CodexNetDeployer
         [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
         public int BlockTTL { get; set; } = SecondsIn1Day;
 
-        [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
-        public bool RecordMetrics { get; set; } = false;
-       
-        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
+        [Uniform("block-mi", "bmi", "BLOCKMI", false, "Block maintenance interval in seconds. Default is 10 minutes.")]
+        public int BlockMI { get; set; } = TenMinutes;
 
+        [Uniform("block-mn", "bmn", "BLOCKMN", false, "Number of blocks maintained per interval. Default is 1000 blocks.")]
+        public int BlockMN { get; set; } = 1000;
+
+        [Uniform("metrics", "m", "METRICS", false, "[None*, Record, Dashboard]. Determines if metrics will be recorded and if a dashboard service will be created.")]
+        public MetricsMode Metrics { get; set; } = MetricsMode.None;
+
+        [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
+            "set this option to override the label value.")]
+        public string TestsTypePodLabel { get; set; } = "continuous-tests";
+
+        [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment.")]
+        public bool CheckPeerConnection { get; set; } = false;
+       
         public List<string> Validate()
         {
             var errors = new List<string>();

CodexNetDeployer/Deployer.cs

@@ -8,29 +8,29 @@ namespace CodexNetDeployer
     public class Deployer
     {
         private readonly Configuration config;
-        private readonly NullLog log;
         private readonly DefaultTimeSet timeset;
+        private readonly PeerConnectivityChecker peerConnectivityChecker;
 
         public Deployer(Configuration config)
         {
             this.config = config;
-            log = new NullLog();
             timeset = new DefaultTimeSet();
+            peerConnectivityChecker = new PeerConnectivityChecker();
         }
 
         public CodexDeployment Deploy()
         {
             Log("Initializing...");
-            var (workflowCreator, lifecycle) = CreateFacilities();
+            var lifecycle = CreateTestLifecycle();
 
             Log("Preparing configuration...");
             // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
             var setup = new CodexSetup(999, config.CodexLogLevel);
             setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
-            setup.MetricsEnabled = config.RecordMetrics;
+            setup.MetricsMode = config.Metrics;
 
             Log("Creating Geth instance and deploying contracts...");
-            var gethStarter = new GethStarter(lifecycle, workflowCreator);
+            var gethStarter = new GethStarter(lifecycle);
             var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
 
             Log("Geth started. Codex contracts deployed.");
@@ -44,20 +44,23 @@ namespace CodexNetDeployer
             Log("Starting Codex nodes...");
 
             // Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
-            var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, gethResults, config.NumberOfValidators!.Value);
-            var codexContainers = new List<RunningContainer>();
+            var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
+            var startResults = new List<CodexNodeStartResult>();
             for (var i = 0; i < config.NumberOfCodexNodes; i++)
             {
-                var container = codexStarter.Start(i);
-                if (container != null) codexContainers.Add(container);
+                var result = codexStarter.Start(i);
+                if (result != null) startResults.Add(result);
             }
 
-            var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers);
+            var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container));
 
-            return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
+            CheckPeerConnectivity(startResults);
+            CheckContainerRestarts(startResults);
+
+            return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
         }
 
-        private (WorkflowCreator, TestLifecycle) CreateFacilities()
+        private TestLifecycle CreateTestLifecycle()
         {
             var kubeConfig = GetKubeConfig(config.KubeConfigFile);
 
@@ -68,28 +71,28 @@ namespace CodexNetDeployer
                 logDebug: false,
                 dataFilesPath: "notUsed",
                 codexLogLevel: config.CodexLogLevel,
-                runnerLocation: config.RunnerLocation
+                k8sNamespacePrefix: config.KubeNamespace
             );
 
-            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: config.KubeNamespace,
-                kubeConfigFile: kubeConfig,
-                operationTimeout: timeset.K8sOperationTimeout(),
-                retryDelay: timeset.WaitForK8sServiceDelay());
-
-            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
-            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
-
-            return (workflowCreator, lifecycle);
+            var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, timeset, string.Empty);
+            DefaultContainerRecipe.TestsType = config.TestsTypePodLabel;
+            DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
+            return lifecycle;
         }
 
-        private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
+        private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable<RunningContainer> codexContainers)
         {
-            if (!setup.MetricsEnabled) return null;
+            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
 
             Log("Starting metrics service...");
-            var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray());
-            return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
+            var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
+            var prometheusContainer = lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
+
+            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null);
+
+            Log("Starting dashboard service...");
+            var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer, setup);
+            return (prometheusContainer, grafanaStartInfo);
         }
 
         private string? GetKubeConfig(string kubeConfigFile)
@@ -98,6 +101,35 @@ namespace CodexNetDeployer
             return kubeConfigFile;
         }
 
+        private void CheckPeerConnectivity(List<CodexNodeStartResult> codexContainers)
+        {
+            if (!config.CheckPeerConnection) return;
+
+            Log("Starting peer-connectivity check for deployed nodes...");
+            peerConnectivityChecker.CheckConnectivity(codexContainers);
+            Log("Check passed.");
+        }
+
+        private void CheckContainerRestarts(List<CodexNodeStartResult> startResults)
+        {
+            var crashes = new List<RunningContainer>();
+            foreach (var startResult in startResults)
+            {
+                var watcher = startResult.Workflow.CreateCrashWatcher(startResult.Container);
+                if (watcher.HasContainerCrashed()) crashes.Add(startResult.Container);
+            }
+
+            if (!crashes.Any())
+            {
+                Log("Container restart check passed.");
+            }
+            else
+            {
+                Log($"Deployment failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}");
+                throw new Exception("Deployment failed: One or more containers crashed.");
+            }
+        }
+
         private DeploymentMetadata CreateMetadata()
         {
             return new DeploymentMetadata(
@@ -109,7 +141,10 @@ namespace CodexNetDeployer
                 initialTestTokens: config.InitialTestTokens,
                 minPrice: config.MinPrice,
                 maxCollateral: config.MaxCollateral,
-                maxDuration: config.MaxDuration);
+                maxDuration: config.MaxDuration,
+                blockTTL: config.BlockTTL,
+                blockMI: config.BlockMI,
+                blockMN: config.BlockMN);
         }
 
         private void Log(string msg)

CodexNetDeployer/PeerConnectivityChecker.cs

@@ -0,0 +1,34 @@
+using DistTestCore.Helpers;
+using Logging;
+
+namespace CodexNetDeployer
+{
+    public class PeerConnectivityChecker
+    {
+        public void CheckConnectivity(List<CodexNodeStartResult> startResults)
+        {
+            var log = new ConsoleLog();
+            var checker = new PeerConnectionTestHelpers(log);
+            var access = startResults.Select(r => r.Access);
+
+            checker.AssertFullyConnected(access);
+        }
+    }
+
+    public class ConsoleLog : BaseLog
+    {
+        public ConsoleLog() : base(false)
+        {
+        }
+
+        protected override string GetFullName()
+        {
+            return "CONSOLE";
+        }
+
+        public override void Log(string message)
+        {
+            Console.WriteLine(message);
+        }
+    }
+}

CodexNetDeployer/Program.cs

@@ -1,6 +1,5 @@
 using ArgsUniform;
 using CodexNetDeployer;
-using DistTestCore;
 using DistTestCore.Codex;
 using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
@@ -17,11 +16,6 @@ public class Program
         var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
         var config = uniformArgs.Parse(true);
         
-        if (args.Any(a => a == "--external"))
-        {
-            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
-        }
-
         var errors = config.Validate();
         if (errors.Any())
         {
@@ -36,7 +30,8 @@ public class Program
             $"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl +
             $"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl +
             $"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl +
-            $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl);
+            $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl +
+            $"\tGrafana image: '{new GrafanaContainerRecipe().Image}'" + nl);
 
         if (!args.Any(a => a == "-y"))
         {
@@ -61,8 +56,5 @@ public class Program
         Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
             "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
             "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
-
-        Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
-            "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
     }
 }

CodexNetDeployer/deploy-continuous-testnet.sh

@@ -9,5 +9,8 @@ dotnet run \
     --min-price=1024 \
     --max-collateral=1024 \
     --max-duration=3600000 \
-    --block-ttl=120 \
-	-y
+    --block-ttl=180 \
+    --block-mi=120 \
+    --block-mn=10000 \
+    --metrics=Dashboard \
+    --check-connect=1

CodexNetDownloader/Configuration.cs

@@ -1,5 +1,4 @@
 using ArgsUniform;
-using DistTestCore;
 using DistTestCore.Codex;
 
 namespace CodexNetDownloader
@@ -16,7 +15,5 @@ namespace CodexNetDownloader
         public string KubeConfigFile { get; set; } = "null";
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
-
-        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
     }
 }

CodexNetDownloader/Program.cs

@@ -15,17 +15,12 @@ public class Program
         var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(PrintHelp, args);
         var config = uniformArgs.Parse(true);
 
-        if (args.Any(a => a == "--external"))
-        {
-            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
-        }
-
         config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
 
         if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
 
         var k8sFactory = new K8sFactory();
-        var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+        var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
 
         foreach (var container in config.CodexDeployment.CodexContainers)
         {

ContinuousTests/CodexAccessFactory.cs

@@ -12,7 +12,7 @@ namespace ContinuousTests
             return containers.Select(container =>
             {
                 var address = container.ClusterExternalAddress;
-                if (config.RunnerLocation == TestRunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
+                if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
                 return new CodexAccess(log, container, timeSet, address);
             }).ToArray();
         }

ContinuousTests/Configuration.cs

@@ -22,15 +22,15 @@ namespace ContinuousTests
         [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
         public string KubeConfigFile { get; set; } = "null";
 
-        [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
-        public bool StopOnFailure { get; set; } = false;
+        [Uniform("stop", "s", "STOPONFAIL", false, "If greater than zero, runner will stop after this many test failures and download all cluster container logs. 0 by default.")]
+        public int StopOnFailure { get; set; } = 0;
 
         [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
         public bool DownloadContainerLogs { get; set; } = false;
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
 
-        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
+        public RunnerLocation RunnerLocation { get; set; }
     }
 
     public class ConfigLoader
@@ -42,10 +42,7 @@ namespace ContinuousTests
             var result = uniformArgs.Parse(true);
             
             result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
-            if (args.Any(a => a == "--external"))
-            {
-                result.RunnerLocation = TestRunnerLocation.ExternalToCluster;
-            }
+            result.RunnerLocation = RunnerLocationUtils.DetermineRunnerLocation(result.CodexDeployment.CodexContainers.First());
 
             return result;
         }
@@ -63,7 +60,6 @@ namespace ContinuousTests
             Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
                 "The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
 
-
             Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
                 "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
         }

ContinuousTests/ContinuousLogDownloader.cs

@@ -1,99 +0,0 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using KubernetesWorkflow;
-
-namespace ContinuousTests
-{
-    public class ContinuousLogDownloader
-    {
-        private readonly TestLifecycle lifecycle;
-        private readonly CodexDeployment deployment;
-        private readonly string outputPath;
-        private readonly CancellationToken cancelToken;
-
-        public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken)
-        {
-            this.lifecycle = lifecycle;
-            this.deployment = deployment;
-            this.outputPath = outputPath;
-            this.cancelToken = cancelToken;
-        }
-
-        public void Run()
-        {
-            while (!cancelToken.IsCancellationRequested)
-            {
-                UpdateLogs();
-
-                cancelToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(15));
-            }
-
-            // After testing has stopped, we wait a little bit and fetch the logs one more time.
-            // If our latest fetch was not recent, interesting test-related log activity might
-            // not have been captured yet.
-            Thread.Sleep(TimeSpan.FromSeconds(10));
-            UpdateLogs();
-        }
-
-        private void UpdateLogs()
-        {
-            foreach (var container in deployment.CodexContainers)
-            {
-                UpdateLog(container);
-            }
-        }
-
-        private void UpdateLog(RunningContainer container)
-        {
-            var filepath = Path.Combine(outputPath, GetLogName(container));
-            if (!File.Exists(filepath))
-            {
-                File.WriteAllLines(filepath, new[] { container.Name });
-            }
-
-            var appender = new LogAppender(filepath);
-
-            lifecycle.CodexStarter.DownloadLog(container, appender);
-        }
-
-        private static string GetLogName(RunningContainer container)
-        {
-            return container.Name
-                .Replace("<", "")
-                .Replace(">", "")
-                + ".log";
-        }
-    }
-
-    public class LogAppender : ILogHandler
-    {
-        private readonly string filename;
-
-        public LogAppender(string filename)
-        {
-            this.filename = filename;
-        }
-
-        public void Log(Stream log)
-        {
-            using var reader = new StreamReader(log);
-            var lines = File.ReadAllLines(filename);
-            var lastLine = lines.Last();
-            var recording = lines.Length < 3;
-            var line = reader.ReadLine();
-            while (line != null)
-            {
-                if (recording)
-                {
-                    File.AppendAllLines(filename, new[] { line });
-                }
-                else
-                {
-                    recording = line == lastLine;
-                }
-
-                line = reader.ReadLine();
-            }
-        }
-    }
-}

ContinuousTests/ContinuousTest.cs

@@ -1,5 +1,7 @@
 using DistTestCore;
 using DistTestCore.Codex;
+using DistTestCore.Logs;
+using KubernetesWorkflow;
 using Logging;
 
 namespace ContinuousTests
@@ -87,6 +89,12 @@ namespace ContinuousTests
             return file;
         }
 
+        public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null)
+        {
+            var nodeRunner = new NodeRunner(Nodes, Configuration, TimeSet, Log, Configuration.CodexDeployment.Metadata.KubeNamespace, EthereumAccountIndex);
+            return nodeRunner.DownloadLog(container, tailLines);
+        }
+
         private void DownloadToFile(CodexAccess node, string contentId, TestFile file)
         {
             using var fileStream = File.OpenWrite(file.Filename);

ContinuousTests/ContinuousTestRunner.cs

@@ -30,9 +30,7 @@ namespace ContinuousTests
 
             ClearAllCustomNamespaces(allTests, overviewLog);
 
-            StartLogDownloader(taskFactory);
-
-            var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
+            var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
 
             foreach (var testLoop in testLoops)
             {
@@ -60,21 +58,8 @@ namespace ContinuousTests
             if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
 
             log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
-            var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
-            workflowCreator.CreateWorkflow().DeleteTestResources();
-        }
-
-        private void StartLogDownloader(TaskFactory taskFactory)
-        {
-            if (!config.DownloadContainerLogs) return;
-
-            var path = Path.Combine(config.LogPath, "containers");
-            if (!Directory.Exists(path)) Directory.CreateDirectory(path);
-
-            var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
-            var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
-
-            taskFactory.Run(downloader.Run);
+            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log);
+            lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources();
         }
     }
 }

ContinuousTests/K8sFactory.cs

@@ -1,13 +1,12 @@
 using DistTestCore.Codex;
 using DistTestCore;
-using KubernetesWorkflow;
 using Logging;
 
 namespace ContinuousTests
 {
     public class K8sFactory
     {
-        public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation)
+        public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log)
         {
             var kubeConfig = GetKubeConfig(kubeConfigFile);
             var lifecycleConfig = new DistTestCore.Configuration
@@ -17,19 +16,13 @@ namespace ContinuousTests
                 logDebug: false,
                 dataFilesPath: dataFilePath,
                 codexLogLevel: CodexLogLevel.Debug,
-                runnerLocation: runnerLocation
+                k8sNamespacePrefix: customNamespace
             );
 
-            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: customNamespace,
-                kubeConfigFile: kubeConfig,
-                operationTimeout: timeSet.K8sOperationTimeout(),
-            retryDelay: timeSet.WaitForK8sServiceDelay());
-
-            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
-            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator);
-
-            return (workflowCreator, lifecycle);
+            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, string.Empty);
+            DefaultContainerRecipe.TestsType = "continuous-tests";
+            DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
+            return lifecycle;
         }
 
         private static string? GetKubeConfig(string kubeConfigFile)

ContinuousTests/NodeRunner.cs

@@ -5,6 +5,7 @@ using KubernetesWorkflow;
 using NUnit.Framework;
 using Logging;
 using Utils;
+using DistTestCore.Logs;
 
 namespace ContinuousTests
 {
@@ -38,10 +39,25 @@ namespace ContinuousTests
             RunNode(bootstrapNode, operation, 0.TestTokens());
         }
 
+        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
+        {
+            var subFile = log.CreateSubfile();
+            var description = container.Name;
+            var handler = new LogDownloadHandler(container, description, subFile);
+
+            log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
+
+            var lifecycle = CreateTestLifecycle();
+            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
+            flow.DownloadContainerLog(container, handler, tailLines);
+
+            return new DownloadedLog(subFile, description);
+        }
+
         public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
         {
-            var (workflowCreator, lifecycle) = CreateFacilities();
-            var flow = workflowCreator.CreateWorkflow();
+            var lifecycle = CreateTestLifecycle();
+            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
 
             try
             {
@@ -89,9 +105,9 @@ namespace ContinuousTests
             }
         }
 
-        private (WorkflowCreator, TestLifecycle) CreateFacilities()
+        private TestLifecycle CreateTestLifecycle()
         {
-            return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation);
+            return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log);
         }
     }
 }

ContinuousTests/SingleTestRun.cs

@@ -23,8 +23,9 @@ namespace ContinuousTests
         private readonly FixtureLog fixtureLog;
         private readonly string testName;
         private readonly string dataFolder;
+        private static int failureCount = 0;
 
-        public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, CancellationToken cancelToken)
+        public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken)
         {
             this.taskFactory = taskFactory;
             this.config = config;
@@ -33,8 +34,9 @@ namespace ContinuousTests
             this.cancelToken = cancelToken;
             testName = handle.Test.GetType().Name;
             fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
+            ApplyLogReplacements(fixtureLog, startupChecker);
 
-            nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
+            nodes = CreateRandomNodes();
             dataFolder = config.DataPath + "-" + Guid.NewGuid();
             fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
         }
@@ -71,16 +73,26 @@ namespace ContinuousTests
                 fixtureLog.Error("Test run failed with exception: " + ex);
                 fixtureLog.MarkAsFailed();
 
-                if (config.StopOnFailure)
+                failureCount++;
+                if (config.StopOnFailure > 0)
                 {
-                    OverviewLog("Configured to stop on first failure. Downloading cluster logs...");
-                    DownloadClusterLogs();
-                    OverviewLog("Log download finished. Cancelling test runner...");
-                    Cancellation.Cts.Cancel();
+                    OverviewLog($"Failures: {failureCount} / {config.StopOnFailure}");
+                    if (failureCount >= config.StopOnFailure)
+                    {
+                        OverviewLog($"Configured to stop after {config.StopOnFailure} failures. Downloading cluster logs...");
+                        DownloadClusterLogs();
+                        OverviewLog("Log download finished. Cancelling test runner...");
+                        Cancellation.Cts.Cancel();
+                    }
                 }
             }
         }
 
+        private void ApplyLogReplacements(FixtureLog fixtureLog, StartupChecker startupChecker)
+        {
+            foreach (var replacement in startupChecker.LogReplacements) fixtureLog.AddStringReplace(replacement.From, replacement.To);
+        }
+
         private void RunTestMoments()
         {
             var earliestMoment = handle.GetEarliestMoment();
@@ -112,7 +124,7 @@ namespace ContinuousTests
                     {
                         ThrowFailTest();
                     }
-                    OverviewLog(" > Test passed. " + FuturesInfo());
+                    OverviewLog(" > Test passed.");
                     return;
                 }
             }
@@ -120,25 +132,19 @@ namespace ContinuousTests
 
         private void ThrowFailTest()
         {
-            var ex = UnpackException(exceptions.First());
-            Log(ex.ToString());
-            OverviewLog($" > Test failed {FuturesInfo()}: " + ex.Message);
-            throw ex;
-        }
-
-        private string FuturesInfo()
-        {
-            var containers = config.CodexDeployment.CodexContainers;
-            var nodes = codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
-            var f = nodes.Select(n => n.GetDebugFutures().ToString());
-            var msg = $"(Futures: [{string.Join(", ", f)}])";
-            return msg;
+            var exs = UnpackExceptions(exceptions);
+            var exceptionsMessage = GetCombinedExceptionsMessage(exs);
+            Log(exceptionsMessage);
+            OverviewLog($" > Test failed: " + exceptionsMessage);
+            throw new Exception(exceptionsMessage);
         }
 
         private void DownloadClusterLogs()
         {
             var k8sFactory = new K8sFactory();
-            var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+            var log = new NullLog();
+            log.FullFilename = Path.Combine(config.LogPath, "NODE");
+            var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), log);
 
             foreach (var container in config.CodexDeployment.CodexContainers)
             {
@@ -146,6 +152,16 @@ namespace ContinuousTests
             }
         }
 
+        private string GetCombinedExceptionsMessage(Exception[] exceptions)
+        {
+            return string.Join(Environment.NewLine, exceptions.Select(ex => ex.ToString()));
+        }
+
+        private Exception[] UnpackExceptions(List<Exception> exceptions)
+        {
+            return exceptions.Select(UnpackException).ToArray();
+        }
+
         private Exception UnpackException(Exception exception)
         {
             if (exception is AggregateException a)
@@ -196,19 +212,28 @@ namespace ContinuousTests
         private void OverviewLog(string msg)
         {
             Log(msg);
-            var containerNames = $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
+            var containerNames = GetContainerNames();
             overviewLog.Log($"{containerNames} {testName}: {msg}");
         }
 
-        private CodexAccess[] CreateRandomNodes(int number)
+        private string GetContainerNames()
         {
-            var containers = SelectRandomContainers(number);
+            if (handle.Test.RequiredNumberOfNodes == -1) return "(All Nodes)";
+            return $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
+        }
+
+        private CodexAccess[] CreateRandomNodes()
+        {
+            var containers = SelectRandomContainers();
             fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
             return codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
         }
 
-        private RunningContainer[] SelectRandomContainers(int number)
+        private RunningContainer[] SelectRandomContainers()
         {
+            var number = handle.Test.RequiredNumberOfNodes;
+            if (number == -1) return config.CodexDeployment.CodexContainers;
+
             var containers = config.CodexDeployment.CodexContainers.ToList();
             var result = new RunningContainer[number];
             for (var i = 0; i < number; i++)
@@ -221,7 +246,7 @@ namespace ContinuousTests
         private DistTestCore.Configuration CreateFileManagerConfiguration()
         {
             return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
-                CodexLogLevel.Error, config.RunnerLocation);
+                CodexLogLevel.Error, string.Empty);
         }
     }
 }

ContinuousTests/StartupChecker.cs

@@ -15,6 +15,7 @@ namespace ContinuousTests
         {
             this.config = config;
             this.cancelToken = cancelToken;
+            LogReplacements = new List<BaseLogStringReplacement>();
         }
 
         public void Check()
@@ -28,6 +29,8 @@ namespace ContinuousTests
             log.Log("All OK.");
         }
 
+        public List<BaseLogStringReplacement> LogReplacements { get; }
+
         private void PreflightCheck(Configuration config)
         {
             var tests = testFactory.CreateTests();
@@ -90,6 +93,7 @@ namespace ContinuousTests
                 if (info == null || string.IsNullOrEmpty(info.id)) return false;
 
                 log.Log($"Codex version: '{info.codex.version}' revision: '{info.codex.revision}'");
+                LogReplacements.Add(new BaseLogStringReplacement(info.id, n.GetName()));
             }
             catch
             {
@@ -156,9 +160,16 @@ namespace ContinuousTests
         {
             foreach (var test in tests)
             {
-                if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
+                if (test.RequiredNumberOfNodes != -1)
                 {
-                    errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
+                    if (test.RequiredNumberOfNodes < 1)
+                    {
+                        errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Test must require > 0 nodes, or -1 to select all nodes.");
+                    }
+                    else if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
+                    {
+                        errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
+                    }
                 }
             }
         }

ContinuousTests/TestLoop.cs

@@ -9,16 +9,18 @@ namespace ContinuousTests
         private readonly BaseLog overviewLog;
         private readonly Type testType;
         private readonly TimeSpan runsEvery;
+        private readonly StartupChecker startupChecker;
         private readonly CancellationToken cancelToken;
         private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset);
 
-        public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, CancellationToken cancelToken)
+        public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken)
         {
             this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
             this.testType = testType;
             this.runsEvery = runsEvery;
+            this.startupChecker = startupChecker;
             this.cancelToken = cancelToken;
             Name = testType.Name;
         }
@@ -58,7 +60,7 @@ namespace ContinuousTests
         {
             var test = (ContinuousTest)Activator.CreateInstance(testType)!;
             var handle = new TestHandle(test);
-            var run = new SingleTestRun(taskFactory, config, overviewLog, handle, cancelToken);
+            var run = new SingleTestRun(taskFactory, config, overviewLog, handle, startupChecker, cancelToken);
 
             runFinishedHandle.Reset();
             run.Run(runFinishedHandle);

ContinuousTests/Tests/FetchTest.cs

@@ -0,0 +1,18 @@
+using DistTestCore.Helpers;
+
+namespace ContinuousTests.Tests
+{
+    public class FetchTest : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => -1;
+        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);
+        public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
+
+        [TestMoment(t: 0)]
+        public void CheckConnectivity()
+        {
+            var checker = new PeerFetchTestHelpers(Log, FileManager);
+            checker.AssertFullFetchInterconnectivity(Nodes);
+        }
+    }
+}

ContinuousTests/Tests/HoldMyBeerTest.cs

@@ -0,0 +1,29 @@
+using DistTestCore;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class HoldMyBeerTest : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => 1;
+        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+
+        private ContentId? cid;
+        private TestFile file = null!;
+
+        [TestMoment(t: Zero)]
+        public void UploadTestFile()
+        {
+            var filesize = 80.MB();
+
+            file = FileManager.GenerateTestFile(filesize);
+
+            cid = UploadFile(Nodes[0], file);
+            Assert.That(cid, Is.Not.Null);
+
+            var dl = DownloadFile(Nodes[0], cid!);
+            file.AssertIsEqual(dl);
+        }
+    }
+}

ContinuousTests/Tests/PeersTest.cs

@@ -0,0 +1,57 @@
+using DistTestCore.Codex;
+using DistTestCore.Helpers;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class PeersTest : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => -1;
+        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);
+        public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
+
+        [TestMoment(t: 0)]
+        public void CheckConnectivity()
+        {
+            var checker = new PeerConnectionTestHelpers(Log);
+            checker.AssertFullyConnected(Nodes);
+        }
+
+        [TestMoment(t: 10)]
+        public void CheckRoutingTables()
+        {
+            var allInfos = Nodes.Select(n =>
+            {
+                var info = n.GetDebugInfo();
+                Log.Log($"{n.GetName()} = {info.table.localNode.nodeId}");
+                Log.AddStringReplace(info.table.localNode.nodeId, n.GetName());
+                return info;
+            }).ToArray();
+
+            var allIds = allInfos.Select(i => i.table.localNode.nodeId).ToArray();
+            var errors = Nodes.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray();
+
+            if (errors.Any())
+            {
+                Assert.Fail(string.Join(Environment.NewLine, errors));
+            }
+        }
+
+        private string AreAllPresent(CodexAccess n, string[] allIds)
+        {
+            var info = n.GetDebugInfo();
+            var known = info.table.nodes.Select(n => n.nodeId).ToArray();
+            var expected = allIds.Where(i => i != info.table.localNode.nodeId).ToArray();
+
+            if (!expected.All(ex => known.Contains(ex)))
+            {
+                var nl = Environment.NewLine;
+                return $"{nl}At node '{info.table.localNode.nodeId}'{nl}" +
+                    $"Not all of{nl}'{string.Join(",", expected)}'{nl}" +
+                    $"were present in routing table:{nl}'{string.Join(",", known)}'";
+            }
+
+            return string.Empty;
+        }
+    }
+}

ContinuousTests/Tests/TwoClientTest.cs

@@ -6,7 +6,7 @@ namespace ContinuousTests.Tests
     public class TwoClientTest : ContinuousTest
     {
         public override int RequiredNumberOfNodes => 2;
-        public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
+        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);
         public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
 
         private ContentId? cid;
@@ -15,7 +15,7 @@ namespace ContinuousTests.Tests
         [TestMoment(t: Zero)]
         public void UploadTestFile()
         {
-            file = FileManager.GenerateTestFile(10.MB());
+            file = FileManager.GenerateTestFile(80.MB());
 
             cid = UploadFile(Nodes[0], file);
             Assert.That(cid, Is.Not.Null);

ContinuousTests/reports/CodexTestNetReport-August2023.md

@@ -0,0 +1,49 @@
+# Codex Continuous Test-net Report
+Date: 05-09-2023
+
+Report for: 08-2023
+
+
+## Test-net Status
+- Start of month: Offline - faulted
+- End of month: Offline - stopped
+
+(Faulted: Tests fail with such frequency that the information gathered does not justify the cost of leaving the test-net running.)
+
+(Stopped: The number of tests that can successfully run on the test-net is not high enough to justify the cost of leaving it running.)
+
+## Deployment Configuration
+Continous Test-net is deployed to the kubernetes cluster with the following configuration:
+
+5x Codex Nodes:
+- Log-level: Trace
+- Storage quota: 2048 MB
+- Storage sell: 1024 MB
+- Min price: 1024
+- Max collateral: 1024
+- Max duration: 3600000 seconds
+- Block-TTL: 180 seconds
+- Block-MI: 120 seconds
+- Block-MN: 10000 blocks
+
+3 of these 5 nodes have:
+- Validator: true
+
+Kubernetes namespace: 'codex-continuous-tests'
+
+## Test Overview
+| Changes        | Test                | Description                                                                                                                                                                                                                                                   | Status      | Results                                                                               |
+|----------------|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|---------------------------------------------------------------------------------------|
+| No change      | Two-client test     | See report for July 2023.                                                                                                                                                                                                                                     | Faulted     | Test reliably fails. Both upload and download failures occur.                         |
+| No change      | Transient-node test | See report for July 2023.                                                                                                                                                                                                                                     | Not running | Test was not run because the previous more rudamentory test has faulted.              |
+| New in 08-2023 | HoldMyBeer test     | Named so because the test is quite dumb and only holds a file, this test selects 1 node at random and uploads an 80 MB file. Then it downloads and verifies it. The purpose of this test is to isolate the upload/download behavior to identify issues in it. | Passed      | Several successful runs. Record run: 48h. After that the test was manually stopped.   |
+| New in 08-2023 | Peers test          | Another attempt to isolate a portion of the two-client test behavior. This test periodically checks the connectivity status between each node in the network. It does this by analysing routing-table information and by performing a node-find operation.    | Passed      | Several successful runs. Record run: 2d21h. After that the test was manually stopped. |
+
+## Resulting changes
+As a result of the testing efforts in 08-2023, these three changes were made:
+1. Block announcement performance - Block iteration (as part of the advertising loop) was found to be performing too much disc IO. Reducing this greatly increased client stability.
+1. Component initialization order - A timing issue in the initialization order of the several Codex components caused nodes to improperly announce their addresses over the network in case the marketplace support was enabled.
+1. Block maintenance performance - Block iteration (again but this time in maintenance) was causing the node to not respond to Ping messages quickly enough, causing other nodes to kick it from the network. By slowing down block maintenance this is solved. A future update of the DHT logic will also impact this behavior.
+
+## Action Points
+- Despite both HoldMyBeer and Peers test success, the two-client test is still not passing. In follow-up actions, we can investigate this issue further by decoupling block retrieval from file download.

ContinuousTests/reports/CodexTestNetReport-July2023.md

@@ -0,0 +1,38 @@
+# Codex Continuous Test-net Report
+Date: 02-08-2023
+
+Report for: 07-2023
+
+
+## Test-net Status
+- Start of month: Offline - faulted
+- End of month: Offline - faulted
+
+(Faulted: Tests fail with such frequency that the information gathered does not justify the cost of leaving the test-net running.)
+
+## Deployment Configuration
+Continous Test-net is deployed to the kubernetes cluster with the following configuration:
+
+5x Codex Nodes:
+- Log-level: Trace
+- Storage quota: 2048 MB
+- Storage sell: 1024 MB
+- Min price: 1024
+- Max collateral: 1024
+- Max duration: 3600000 seconds
+- Block-TTL: 120 seconds
+
+3 of these 5 nodes have:
+- Validator: true
+
+Kubernetes namespace: 'codex-continuous-tests'
+
+## Test Overview
+| Changes        | Test                | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | Status      | Results                                                                              |
+|----------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|--------------------------------------------------------------------------------------|
+| New in 07-2023 | Two-client test     | Every 30 seconds, two nodes are chosen at random. A 10 MB file is generated and uploaded to one. 10 seconds later, it is downloaded from the other. File contents are asserted to be equal.                                                                                                                                                                                                                                                                                                                                                                                    | Faulted     | Test reliably fails after 30 to 45 minutes. Both upload and download failures occur. |
+| New in 07-2023 | Transient-node test | Every 1 minute, a new, transient Codex node is started and bootstrapped against a random node of the test net. A 10 MB file is generated and uploaded to the transient node. The file is then downloaded from a second random test net node and file equality is asserted. After that, the transient node is shut down. 30 seconds later, a new transient Codex node is started and bootstrapped against a third (guaranteed different from first and second) node from the test net. The same file is then downloaded from the new transient node. File equality is asserted. | Not running | Test was not run because the previous more rudamentory test has faulted.             |
+
+## Action Points
+- Codex logs (from these long-running containers) are often incomplete when downloaded after a test failure. A reliable way of maintaining these logs is needed. The logs can quickly explode in size, even for test-nets that run only for a few hours.
+- The distributed testing setup can and has been used to reproduce the failure of the Two-client test in a local environment. Investigation is on-going.

ContinuousTests/run.sh

@@ -2,5 +2,5 @@ dotnet run \
     --kube-config=/opt/kubeconfig.yaml \
     --codex-deployment=codex-deployment.json \
     --keep=1 \
-    --stop=1 \
+    --stop=10 \
     --dl-logs=1

DistTestCore/BaseStarter.cs

@@ -1,18 +1,15 @@
-using KubernetesWorkflow;
-using Logging;
+using Logging;
 
 namespace DistTestCore
 {
     public class BaseStarter
     {
         protected readonly TestLifecycle lifecycle;
-        protected readonly WorkflowCreator workflowCreator;
         private Stopwatch? stopwatch;
 
-        public BaseStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+        public BaseStarter(TestLifecycle lifecycle)
         {
             this.lifecycle = lifecycle;
-            this.workflowCreator = workflowCreator;
         }
 
         protected void LogStart(string msg)

DistTestCore/ByteSize.cs

@@ -38,6 +38,11 @@ namespace DistTestCore
     {
         private const long Kilo = 1024;
 
+        public static ByteSize Bytes(this long i)
+        {
+            return new ByteSize(i);
+        }
+
         public static ByteSize KB(this long i)
         {
             return new ByteSize(i * Kilo);
@@ -58,6 +63,11 @@ namespace DistTestCore
             return (i * Kilo).GB();
         }
 
+        public static ByteSize Bytes(this int i)
+        {
+            return new ByteSize(i);
+        }
+
         public static ByteSize KB(this int i)
         {
             return Convert.ToInt64(i).KB();

DistTestCore/Codex/CodexAccess.cs

@@ -4,10 +4,11 @@ using Utils;
 
 namespace DistTestCore.Codex
 {
-    public class CodexAccess
+    public class CodexAccess : ILogHandler
     {
         private readonly BaseLog log;
         private readonly ITimeSet timeSet;
+        private bool hasContainerCrashed;
 
         public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
         {
@@ -15,6 +16,9 @@ namespace DistTestCore.Codex
             Container = container;
             this.timeSet = timeSet;
             Address = address;
+            hasContainerCrashed = false;
+
+            if (container.CrashWatcher != null) container.CrashWatcher.Start(this);
         }
 
         public RunningContainer Container { get; }
@@ -43,10 +47,10 @@ namespace DistTestCore.Codex
             return result;
         }
 
-        public int GetDebugFutures()
+        public CodexDebugFetchResponse DebugFetch(string contentId)
         {
-            // Some Codex images support debug/futures to count the number of open futures.
-            return 0; // Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
+            var http = Http();
+            return http.HttpGetJson<CodexDebugFetchResponse>($"debug/fetch/{contentId}");
         }
 
         public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
@@ -84,9 +88,36 @@ namespace DistTestCore.Codex
             return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
         }
 
+        public string GetName()
+        {
+            return Container.Name;
+        }
+
         private Http Http()
         {
-            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name);
+            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
+        }
+
+        private void CheckContainerCrashed(HttpClient client)
+        {
+            if (hasContainerCrashed) throw new Exception("Container has crashed.");
+        }
+
+        public void Log(Stream crashLog)
+        {
+            var file = log.CreateSubfile();
+            log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
+
+            using var reader = new StreamReader(crashLog);
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                file.Write(line);
+                line = reader.ReadLine();
+            }
+
+            log.Log("Crash log successfully downloaded.");
+            hasContainerCrashed = true;
         }
     }
 }

DistTestCore/Codex/CodexApiTypes.cs

@@ -1,7 +1,4 @@
-using KubernetesWorkflow;
-using Logging;
-using Newtonsoft.Json;
-using Utils;
+using Newtonsoft.Json;
 
 namespace DistTestCore.Codex
 {
@@ -87,6 +84,23 @@ namespace DistTestCore.Codex
         public string address { get; set; } = string.Empty;
     }
 
+    public class CodexDebugFetchResponse
+    {
+        public string originalBytes { get; set; } = string.Empty;
+        public string blockSize { get; set; } = string.Empty;
+        public string numberOfBlocks { get; set; } = string.Empty;
+        public string version { get; set; } = string.Empty;
+        public string hcodec { get; set; } = string.Empty;
+        public string codec { get; set; } = string.Empty;
+        public string @protected { get; set; } = string.Empty;
+        public CodexDebugFetchBlockResponse[] blocks { get; set; } = Array.Empty<CodexDebugFetchBlockResponse>();
+    }
+
+    public class CodexDebugFetchBlockResponse
+    {
+        public string cid { get; set; } = string.Empty;
+    }
+
     public class CodexDebugThresholdBreaches
     {
         public string[] breaches { get; set; } = Array.Empty<string>();

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -3,17 +3,19 @@ using KubernetesWorkflow;
 
 namespace DistTestCore.Codex
 {
-    public class CodexContainerRecipe : ContainerRecipeFactory
+    public class CodexContainerRecipe : DefaultContainerRecipe
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:latest";
+        //private const string DefaultDockerImage = "codexstorage/nim-codex:sha-d279eeb-dist-tests";
+        private const string DefaultDockerImage = "thatbenbierens/nim-codex:debugfetch";
 
         public const string MetricsPortTag = "metrics_port";
         public const string DiscoveryPortTag = "discovery-port";
 
-        // Used by tests for time-constraint assersions.
+        // Used by tests for time-constraint assertions.
         public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
         public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
 
+        public override string AppName => "codex";
         public override string Image { get; }
 
         public CodexContainerRecipe()
@@ -21,7 +23,7 @@ namespace DistTestCore.Codex
             Image = GetDockerImage();
         }
 
-        protected override void Initialize(StartupConfig startupConfig)
+        protected override void InitializeRecipe(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<CodexStartupConfig>();
 
@@ -50,11 +52,22 @@ namespace DistTestCore.Codex
             {
                 AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
             }
-            if (config.MetricsEnabled)
+            if (config.BlockMaintenanceInterval != null)
             {
+                AddEnvVar("CODEX_BLOCK_MI", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
+            }
+            if (config.BlockMaintenanceNumber != null)
+            {
+                AddEnvVar("CODEX_BLOCK_MN", config.BlockMaintenanceNumber.ToString()!);
+            }
+            if (config.MetricsMode != Metrics.MetricsMode.None)
+            {
+                var metricsPort = AddInternalPort(MetricsPortTag);
                 AddEnvVar("CODEX_METRICS", "true");
                 AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
-                AddInternalPortAndVar("CODEX_METRICS_PORT", tag: MetricsPortTag);
+                AddEnvVar("CODEX_METRICS_PORT", metricsPort);
+                AddPodAnnotation("prometheus.io/scrape", "true");
+                AddPodAnnotation("prometheus.io/port", metricsPort.Number.ToString());
             }
 
             if (config.MarketplaceConfig != null)

DistTestCore/Codex/CodexDeployment.cs

@@ -5,23 +5,25 @@ namespace DistTestCore.Codex
 {
     public class CodexDeployment
     {
-        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata)
+        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, GrafanaStartInfo? grafanaStartInfo, DeploymentMetadata metadata)
         {
             GethStartResult = gethStartResult;
             CodexContainers = codexContainers;
             PrometheusContainer = prometheusContainer;
+            GrafanaStartInfo = grafanaStartInfo;
             Metadata = metadata;
         }
 
         public GethStartResult GethStartResult { get; }
         public RunningContainer[] CodexContainers { get; }
         public RunningContainer? PrometheusContainer { get; }
+        public GrafanaStartInfo? GrafanaStartInfo { get; }
         public DeploymentMetadata Metadata { get; }
     }
 
     public class DeploymentMetadata
     {
-        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration)
+        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
         {
             DeployDateTimeUtc = DateTime.UtcNow;
             KubeNamespace = kubeNamespace;
@@ -33,6 +35,9 @@ namespace DistTestCore.Codex
             MinPrice = minPrice;
             MaxCollateral = maxCollateral;
             MaxDuration = maxDuration;
+            BlockTTL = blockTTL;
+            BlockMI = blockMI;
+            BlockMN = blockMN;
         }
 
         public DateTime DeployDateTimeUtc { get; }
@@ -45,5 +50,8 @@ namespace DistTestCore.Codex
         public int MinPrice { get; }
         public int MaxCollateral { get; }
         public int MaxDuration { get; }
+        public int BlockTTL { get; }
+        public int BlockMI { get; }
+        public int BlockMN { get; }
     }
 }

DistTestCore/Codex/CodexStartupConfig.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
 using KubernetesWorkflow;
 
 namespace DistTestCore.Codex
@@ -14,9 +15,11 @@ namespace DistTestCore.Codex
         public Location Location { get; set; }
         public CodexLogLevel LogLevel { get; }
         public ByteSize? StorageQuota { get; set; }
-        public bool MetricsEnabled { get; set; }
+        public MetricsMode MetricsMode { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
         public string? BootstrapSpr { get; set; }
         public int? BlockTTL { get; set; }
+        public TimeSpan? BlockMaintenanceInterval { get; set; }
+        public int? BlockMaintenanceNumber { get; set; }
     }
 }

DistTestCore/CodexNodeGroup.cs

@@ -14,12 +14,12 @@ namespace DistTestCore
     {
         private readonly TestLifecycle lifecycle;
 
-        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
         {
             this.lifecycle = lifecycle;
             Setup = setup;
             Containers = containers;
-            Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
+            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
             Version = new CodexDebugVersionResponse();
         }
 
@@ -45,7 +45,7 @@ namespace DistTestCore
         }
 
         public CodexSetup Setup { get; private set; }
-        public RunningContainers Containers { get; private set; }
+        public RunningContainers[] Containers { get; private set; }
         public OnlineCodexNode[] Nodes { get; private set; }
         public CodexDebugVersionResponse Version { get; private set; }
 

DistTestCore/CodexSetup.cs

@@ -10,6 +10,9 @@ namespace DistTestCore
         ICodexSetup At(Location location);
         ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
         ICodexSetup WithStorageQuota(ByteSize storageQuota);
+        ICodexSetup WithBlockTTL(TimeSpan duration);
+        ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration);
+        ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks);
         ICodexSetup EnableMetrics();
         ICodexSetup EnableMarketplace(TestToken initialBalance);
         ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
@@ -50,9 +53,27 @@ namespace DistTestCore
             return this;
         }
 
+        public ICodexSetup WithBlockTTL(TimeSpan duration)
+        {
+            BlockTTL = Convert.ToInt32(duration.TotalSeconds);
+            return this;
+        }
+
+        public ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration)
+        {
+            BlockMaintenanceInterval = duration;
+            return this;
+        }
+
+        public ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks)
+        {
+            BlockMaintenanceNumber = numberOfBlocks;
+            return this;
+        }
+
         public ICodexSetup EnableMetrics()
         {
-            MetricsEnabled = true;
+            MetricsMode = Metrics.MetricsMode.Record;
             return this;
         }
 

DistTestCore/CodexStarter.cs

@@ -8,8 +8,8 @@ namespace DistTestCore
 {
     public class CodexStarter : BaseStarter
     {
-        public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public CodexStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
@@ -32,12 +32,12 @@ namespace DistTestCore
             var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
             lifecycle.SetCodexVersion(group.Version);
 
-            var podInfo = group.Containers.RunningPod.PodInfo;
+            var nl = Environment.NewLine;
+            var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
             LogEnd($"Started {codexSetup.NumberOfNodes} nodes " +
-                $"of image '{containers.Containers.First().Recipe.Image}' " +
-                $"and version '{group.Version}' " +
-                $"at location '{podInfo.K8SNodeName}'={podInfo.Ip}. " +
-                $"They are: {group.Describe()}");
+                $"of image '{containers.Containers().First().Recipe.Image}' " +
+                $"and version '{group.Version}'{nl}" +
+                podInfos);
             LogSeparator();
 
             return group;
@@ -47,7 +47,11 @@ namespace DistTestCore
         {
             LogStart($"Stopping {group.Describe()}...");
             var workflow = CreateWorkflow();
-            workflow.Stop(group.Containers);
+            foreach (var c in group.Containers)
+            {
+                StopCrashWatcher(c);
+                workflow.Stop(c);
+            }
             RunningGroups.Remove(group);
             LogEnd("Stopped.");
         }
@@ -60,17 +64,23 @@ namespace DistTestCore
             RunningGroups.Clear();
         }
 
-        public void DownloadLog(RunningContainer container, ILogHandler logHandler)
+        public void DownloadLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
         {
             var workflow = CreateWorkflow();
-            workflow.DownloadContainerLog(container, logHandler);
+            workflow.DownloadContainerLog(container, logHandler, tailLines);
         }
 
-        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers containers)
+        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
         {
-            if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
+            if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory();
 
             var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
+
+            if (codexSetup.MetricsMode == MetricsMode.Dashboard)
+            {
+                lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup);
+            }
+
             return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
         }
 
@@ -83,28 +93,66 @@ namespace DistTestCore
             return startupConfig;
         }
 
-        private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
+        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
         {
-            var workflow = CreateWorkflow();
-            return workflow.Start(numberOfNodes, location, new CodexContainerRecipe(), startupConfig);
+            var result = new List<RunningContainers>();
+            var recipe = new CodexContainerRecipe();
+            for (var i = 0; i < numberOfNodes; i++)
+            {
+                var workflow = CreateWorkflow();
+                var rc = workflow.Start(1, location, recipe, startupConfig);
+                CreateCrashWatcher(workflow, rc);
+                result.Add(rc);
+            }
+            return result.ToArray();
         }
 
-        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory)
+        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
             RunningGroups.Add(group);
-            Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
+
+            try
+            {
+                Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
+            }
+            catch
+            {
+                CodexNodesNotOnline(runningContainers);
+                throw;
+            }
+
             return group;
         }
 
+        private void CodexNodesNotOnline(RunningContainers[] runningContainers)
+        {
+            Log("Codex nodes failed to start");
+            foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
+        }
+
         private StartupWorkflow CreateWorkflow()
         {
-            return workflowCreator.CreateWorkflow();
+            return lifecycle.WorkflowCreator.CreateWorkflow();
         }
 
         private void LogSeparator()
         {
             Log("----------------------------------------------------------------------------");
         }
+
+        private void CreateCrashWatcher(StartupWorkflow workflow, RunningContainers rc)
+        {
+            var c = rc.Containers.Single();
+            c.CrashWatcher = workflow.CreateCrashWatcher(c);
+        }
+
+        private void StopCrashWatcher(RunningContainers containers)
+        {
+            foreach (var c in containers.Containers)
+            {
+                c.CrashWatcher?.Stop();
+            }
+        }
     }
 }

DistTestCore/Configuration.cs

@@ -1,5 +1,6 @@
 using DistTestCore.Codex;
 using KubernetesWorkflow;
+using System.Net.NetworkInformation;
 using Utils;
 
 namespace DistTestCore
@@ -11,7 +12,8 @@ namespace DistTestCore
         private readonly bool logDebug;
         private readonly string dataFilesPath;
         private readonly CodexLogLevel codexLogLevel;
-        private readonly TestRunnerLocation runnerLocation;
+        private readonly string k8sNamespacePrefix;
+        private static RunnerLocation? runnerLocation = null;
 
         public Configuration()
         {
@@ -20,23 +22,23 @@ namespace DistTestCore
             logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
             dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
             codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
-            runnerLocation = ParseEnum.Parse<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
+            k8sNamespacePrefix = "ct-";
         }
 
-        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation)
+        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, string k8sNamespacePrefix)
         {
             this.kubeConfigFile = kubeConfigFile;
             this.logPath = logPath;
             this.logDebug = logDebug;
             this.dataFilesPath = dataFilesPath;
             this.codexLogLevel = codexLogLevel;
-            this.runnerLocation = runnerLocation;
+            this.k8sNamespacePrefix = k8sNamespacePrefix;
         }
 
         public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
         {
             return new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: "ct-",
+                k8sNamespacePrefix: k8sNamespacePrefix,
                 kubeConfigFile: kubeConfigFile,
                 operationTimeout: timeSet.K8sOperationTimeout(),
                 retryDelay: timeSet.WaitForK8sServiceDelay()
@@ -58,14 +60,14 @@ namespace DistTestCore
             return codexLogLevel;
         }
 
-        public TestRunnerLocation GetTestRunnerLocation()
-        {
-            return runnerLocation;
-        }
-
         public Address GetAddress(RunningContainer container)
         {
-            if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster)
+            if (runnerLocation == null)
+            {
+                runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container);
+            }
+            
+            if (runnerLocation == RunnerLocation.InternalToCluster)
             {
                 return container.ClusterInternalAddress;
             }
@@ -87,9 +89,56 @@ namespace DistTestCore
         }
     }
 
-    public enum TestRunnerLocation
+    public enum RunnerLocation
     {
         ExternalToCluster,
         InternalToCluster,
     }
+
+    public static class RunnerLocationUtils
+    {
+        private static bool alreadyDidThat = false;
+
+        public static RunnerLocation DetermineRunnerLocation(RunningContainer container)
+        {
+            // We want to be sure we don't ping more often than strictly necessary.
+            // If we have already determined the location during this application
+            // lifetime, don't do it again.
+            if (alreadyDidThat) throw new Exception("We already did that.");
+            alreadyDidThat = true;
+
+            if (PingHost(container.Pod.PodInfo.Ip))
+            {
+                return RunnerLocation.InternalToCluster;
+            }
+            if (PingHost(Format(container.ClusterExternalAddress)))
+            {
+                return RunnerLocation.ExternalToCluster;
+            }
+
+            throw new Exception("Unable to determine runner location.");
+        }
+
+        private static string Format(Address host)
+        {
+            return host.Host
+                .Replace("http://", "")
+                .Replace("https://", "");
+        }
+
+        private static bool PingHost(string host)
+        {
+            try
+            {
+                using var pinger = new Ping();
+                PingReply reply = pinger.Send(host);
+                return reply.Status == IPStatus.Success;
+            }
+            catch (PingException)
+            {
+            }
+
+            return false;
+        }
+    }
 }

DistTestCore/DefaultContainerRecipe.cs

@@ -0,0 +1,40 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace DistTestCore
+{
+    public abstract class DefaultContainerRecipe : ContainerRecipeFactory
+    {
+        public static string TestsType { get; set; } = "NotSet";
+        public static ApplicationIds? ApplicationIds { get; set; } = null;
+
+        protected abstract void InitializeRecipe(StartupConfig config);
+
+        protected override void Initialize(StartupConfig config)
+        {
+            Add("tests-type", TestsType);
+            Add("runid", NameUtils.GetRunId());
+            Add("testid", NameUtils.GetTestId());
+            Add("category", NameUtils.GetCategoryName());
+            Add("fixturename", NameUtils.GetRawFixtureName());
+            Add("testname", NameUtils.GetTestMethodName());
+
+            if (ApplicationIds != null)
+            {
+                Add("codexid", ApplicationIds.CodexId);
+                Add("gethid", ApplicationIds.GethId);
+                Add("prometheusid", ApplicationIds.PrometheusId);
+                Add("codexcontractsid", ApplicationIds.CodexContractsId);
+                Add("grafanaid", ApplicationIds.GrafanaId);
+            }
+            Add("app", AppName);
+
+            InitializeRecipe(config);
+        }
+
+        private void Add(string name, string value)
+        {
+            AddPodLabel(name, value);
+        }
+    }
+}

DistTestCore/DistTest.cs

@@ -13,6 +13,7 @@ namespace DistTestCore
     [Parallelizable(ParallelScope.All)]
     public abstract class DistTest
     {
+        private const string TestsType = "dist-tests";
         private readonly Configuration configuration = new Configuration();
         private readonly Assembly[] testAssemblies;
         private readonly FixtureLog fixtureLog;
@@ -29,14 +30,8 @@ namespace DistTestCore
             var startTime = DateTime.UtcNow;
             fixtureLog = new FixtureLog(logConfig, startTime);
             statusLog = new StatusLog(logConfig, startTime);
-
-            PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
-            PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
         }
 
-        public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; }
-        public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; }
-
         [OneTimeSetUp]
         public void GlobalSetup()
         {
@@ -52,7 +47,7 @@ namespace DistTestCore
             {
                 Stopwatch.Measure(fixtureLog, "Global setup", () =>
                 {
-                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
+                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), string.Empty);
                     wc.CreateWorkflow().DeleteAllResources();
                 });                
             }
@@ -174,6 +169,26 @@ namespace DistTestCore
             GetTestLog().Debug(msg);
         }
 
+        public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers()
+        {
+            return new PeerConnectionTestHelpers(GetTestLog());
+        }
+
+        public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers()
+        {
+            return new PeerDownloadTestHelpers(GetTestLog(), Get().FileManager);
+        }
+
+        public PeerFetchTestHelpers CreatePeerFetchTestHelpers()
+        {
+            return new PeerFetchTestHelpers(GetTestLog(), Get().FileManager);
+        }
+
+        public void Measure(string name, Action action)
+        {
+            Stopwatch.Measure(Get().Log, name, action);
+        }
+
         protected CodexSetup CreateCodexSetup(int numberOfNodes)
         {
             return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
@@ -195,7 +210,11 @@ namespace DistTestCore
             {
                 lock (lifecycleLock)
                 {
-                    lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
+                    var testNamespace = Guid.NewGuid().ToString();
+                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace);
+                    lifecycles.Add(testName, lifecycle);
+                    DefaultContainerRecipe.TestsType = TestsType;
+                    DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
                 }
             });
         }
@@ -206,7 +225,7 @@ namespace DistTestCore
             var testResult = GetTestResult();
             var testDuration = lifecycle.GetTestDuration();
             fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
-            statusLog.ConcludeTest(testResult, testDuration, GetCodexId(lifecycle));
+            statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetApplicationIds());
             Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
             {
                 lifecycle.Log.EndTest();
@@ -216,14 +235,6 @@ namespace DistTestCore
             });
         }
 
-        private static string GetCodexId(TestLifecycle lifecycle)
-        {
-            var v = lifecycle.CodexVersion;
-            if (v == null) return new CodexContainerRecipe().Image;
-            if (v.version != "untagged build") return v.version;
-            return v.revision;
-        }
-
         private ITimeSet GetTimeSet()
         {
             if (ShouldUseLongTimeouts()) return new LongTimeSet();

DistTestCore/DistTestCore.csproj

@@ -10,6 +10,14 @@
   <PropertyGroup Condition="'$(IsArm64)'=='true'">
     <DefineConstants>Arm64</DefineConstants>
   </PropertyGroup>
+  <ItemGroup>
+    <None Remove="Metrics\dashboard.json" />
+  </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="Metrics\dashboard.json">
+      <CopyToOutputDirectory>Never</CopyToOutputDirectory>
+    </EmbeddedResource>
+  </ItemGroup>
 
   <ItemGroup>
 	  <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />

DistTestCore/GethStarter.cs

@@ -1,5 +1,4 @@
 using DistTestCore.Marketplace;
-using KubernetesWorkflow;
 
 namespace DistTestCore
 {
@@ -8,13 +7,13 @@ namespace DistTestCore
         private readonly MarketplaceNetworkCache marketplaceNetworkCache;
         private readonly GethCompanionNodeStarter companionNodeStarter;
 
-        public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public GethStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
             marketplaceNetworkCache = new MarketplaceNetworkCache(
-                new GethBootstrapNodeStarter(lifecycle, workflowCreator),
-                new CodexContractsStarter(lifecycle, workflowCreator));
-            companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator);
+                new GethBootstrapNodeStarter(lifecycle),
+                new CodexContractsStarter(lifecycle));
+            companionNodeStarter = new GethCompanionNodeStarter(lifecycle);
         }
 
         public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup)

DistTestCore/GrafanaStarter.cs

@@ -0,0 +1,192 @@
+using DistTestCore.Metrics;
+using IdentityModel.Client;
+using KubernetesWorkflow;
+using Newtonsoft.Json;
+using System.Reflection;
+
+namespace DistTestCore
+{
+    public class GrafanaStarter : BaseStarter
+    {
+        private const string StorageQuotaThresholdReplaceToken = "\"<CODEX_STORAGEQUOTA>\"";
+        private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"<CODEX_BYTESUSED_SOFTMAX>\"";
+
+        public GrafanaStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
+        {
+        }
+
+        public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup)
+        {
+            LogStart($"Starting dashboard server");
+
+            var grafanaContainer = StartGrafanaContainer();
+            var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer);
+
+            var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth);
+
+            Log("Connecting datasource...");
+            AddDataSource(http, prometheusContainer);
+
+            Log("Uploading dashboard configurations...");
+            var jsons = ReadEachDashboardJsonFile(codexSetup);
+            var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
+
+            LogEnd("Dashboard server started.");
+
+            return new GrafanaStartInfo(dashboardUrls, grafanaContainer);
+        }
+
+        private RunningContainer StartGrafanaContainer()
+        {
+            var startupConfig = new StartupConfig();
+
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig);
+            if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created.");
+
+            return grafanaContainers.Containers.First();
+        }
+
+        private void AddBasicAuth(HttpClient client)
+        {
+            client.SetBasicAuthentication(
+                GrafanaContainerRecipe.DefaultAdminUser,
+                GrafanaContainerRecipe.DefaultAdminPassword);
+        }
+
+        private static void AddDataSource(Http http, RunningContainer prometheusContainer)
+        {
+            var prometheusAddress = prometheusContainer.ClusterExternalAddress;
+            var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port;
+            var response = http.HttpPostJson<GrafanaDataSourceRequest, GrafanaDataSourceResponse>("datasources", new GrafanaDataSourceRequest
+            {
+                uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb",
+                name = "CodexPrometheus",
+                type = "prometheus",
+                url = prometheusUrl,
+                access = "proxy",
+                basicAuth = false,
+                jsonData = new GrafanaDataSourceJsonData
+                {
+                    httpMethod = "POST"
+                }
+            });
+
+            if (response.message != "Datasource added")
+            {
+                throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message);
+            }
+        }
+
+        public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson)
+        {
+            var request = GetDashboardCreateRequest(dashboardJson);
+            var response = http.HttpPostString("dashboards/db", request);
+            var jsonResponse = JsonConvert.DeserializeObject<GrafanaPostDashboardResponse>(response);
+            if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard.");
+
+            var grafanaAddress = grafanaContainer.ClusterExternalAddress;
+            return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
+        }
+
+        private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup)
+        {
+            var assembly = Assembly.GetExecutingAssembly();
+            var resourceNames = new[]
+            {
+                "DistTestCore.Metrics.dashboard.json"
+            };
+
+            return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray();
+        }
+
+        private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup)
+        {
+            using var stream = assembly.GetManifestResourceStream(resourceName);
+            if (stream == null) throw new Exception("Unable to find resource " + resourceName);
+            using var reader = new StreamReader(stream);
+            return ApplyReplacements(reader.ReadToEnd(), codexSetup);
+        }
+
+        private static string ApplyReplacements(string input, CodexSetup codexSetup)
+        {
+            var quotaString = GetQuotaString(codexSetup);
+            var softMaxString = GetSoftMaxString(codexSetup);
+
+            return input
+                .Replace(StorageQuotaThresholdReplaceToken, quotaString)
+                .Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString);
+        }
+
+        private static string GetQuotaString(CodexSetup codexSetup)
+        {
+            return GetCodexStorageQuotaInBytes(codexSetup).ToString();
+        }
+
+        private static string GetSoftMaxString(CodexSetup codexSetup)
+        {
+            var quota = GetCodexStorageQuotaInBytes(codexSetup);
+            var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing.
+            return softMax.ToString();
+        }
+
+        private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup)
+        {
+            if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes;
+
+            // Codex default: 8GB
+            return 8.GB().SizeInBytes;
+        }
+
+        private static string GetDashboardCreateRequest(string dashboardJson)
+        {
+            return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}";
+        }
+    }
+
+    public class GrafanaStartInfo
+    {
+        public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container)
+        {
+            DashboardUrls = dashboardUrls;
+            Container = container;
+        }
+
+        public string[] DashboardUrls { get; }
+        public RunningContainer Container { get; }
+    }
+
+    public class GrafanaDataSourceRequest
+    {
+        public string uid { get; set; } = string.Empty;
+        public string name { get; set; } = string.Empty;
+        public string type { get; set; } = string.Empty;
+        public string url { get; set; } = string.Empty;
+        public string access { get; set; } = string.Empty;
+        public bool basicAuth { get; set; }
+        public GrafanaDataSourceJsonData jsonData { get; set; } = new();
+    }
+
+    public class GrafanaDataSourceResponse
+    {
+        public int id { get; set; }
+        public string message { get; set; } = string.Empty;
+        public string name { get; set; } = string.Empty;
+    }
+
+    public class GrafanaDataSourceJsonData
+    {
+        public string httpMethod { get; set; } = string.Empty;
+    }
+
+    public class GrafanaPostDashboardResponse
+    {
+        public int id { get; set; }
+        public string slug { get; set; } = string.Empty;
+        public string status { get; set; } = string.Empty;
+        public string uid { get; set; } = string.Empty;
+        public string url { get; set; } = string.Empty;
+        public int version { get; set; }
+    }
+}

DistTestCore/Helpers/FullConnectivityHelper.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Codex;
+using Logging;
 using NUnit.Framework;
 using Utils;
 
@@ -14,23 +15,23 @@ namespace DistTestCore.Helpers
     public class FullConnectivityHelper
     {
         private static string Nl = Environment.NewLine;
-        private readonly DistTest test;
+        private readonly BaseLog log;
         private readonly IFullConnectivityImplementation implementation;
 
-        public FullConnectivityHelper(DistTest test, IFullConnectivityImplementation implementation)
+        public FullConnectivityHelper(BaseLog log, IFullConnectivityImplementation implementation)
         {
-            this.test = test;
+            this.log = log;
             this.implementation = implementation;
         }
 
-        public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
+        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
         {
             AssertFullyConnected(nodes.ToArray());
         }
 
-        private void AssertFullyConnected(IOnlineCodexNode[] nodes)
+        private void AssertFullyConnected(CodexAccess[] nodes)
         {
-            test.Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
             var entries = CreateEntries(nodes);
             var pairs = CreatePairs(entries);
 
@@ -43,19 +44,19 @@ namespace DistTestCore.Helpers
             {
                 var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
 
-                test.Log($"Connections failed:{Nl}{pairDetails}");
+                Log($"Connections failed:{Nl}{pairDetails}");
 
                 Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
             }
             else
             {
-                test.Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
+                Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
             }
         }
 
         private static void RetryWhilePairs(List<Pair> pairs, Action action)
         {
-            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2);
             while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
             {
                 action();
@@ -72,7 +73,7 @@ namespace DistTestCore.Helpers
 
             foreach (var pair in selectedPair)
             {
-                test.ScopedTestFiles(pair.Check);
+                pair.Check();
 
                 if (pair.Success)
                 {
@@ -81,10 +82,10 @@ namespace DistTestCore.Helpers
                 }
             }
 
-            test.Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
+            Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
         }
 
-        private Entry[] CreateEntries(IOnlineCodexNode[] nodes)
+        private Entry[] CreateEntries(CodexAccess[] nodes)
         {
             var entries = nodes.Select(n => new Entry(n)).ToArray();
 
@@ -117,15 +118,20 @@ namespace DistTestCore.Helpers
             }
         }
 
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+
         public class Entry
         {
-            public Entry(IOnlineCodexNode node)
+            public Entry(CodexAccess node)
             {
                 Node = node;
                 Response = node.GetDebugInfo();
             }
 
-            public IOnlineCodexNode Node { get; }
+            public CodexAccess Node { get; }
             public CodexDebugResponse Response { get; }
 
             public override string ToString()

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Codex;
+using Logging;
 using static DistTestCore.Helpers.FullConnectivityHelper;
 
 namespace DistTestCore.Helpers
@@ -7,12 +8,17 @@ namespace DistTestCore.Helpers
     {
         private readonly FullConnectivityHelper helper;
 
-        public PeerConnectionTestHelpers(DistTest test)
+        public PeerConnectionTestHelpers(BaseLog log)
         {
-            helper = new FullConnectivityHelper(test, this);
+            helper = new FullConnectivityHelper(log, this);
         }
 
         public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullyConnected(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess));
+        }
+
+        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
         {
             helper.AssertFullyConnected(nodes);
         }
@@ -57,9 +63,8 @@ namespace DistTestCore.Helpers
             var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
             if (peer == null) return $"peerId: {node.peerId} is not known.";
 
-            var n = (OnlineCodexNode)peer.Node;
-            var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
-            var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
+            var ip = peer.Node.Container.Pod.PodInfo.Ip;
+            var discPort = peer.Node.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
             return $"{ip}:{discPort.Number}";
         }
     }

DistTestCore/Helpers/PeerDownloadTestHelpers.cs

@@ -1,21 +1,30 @@
-using static DistTestCore.Helpers.FullConnectivityHelper;
+using DistTestCore.Codex;
+using Logging;
+using static DistTestCore.Helpers.FullConnectivityHelper;
 
 namespace DistTestCore.Helpers
 {
     public class PeerDownloadTestHelpers : IFullConnectivityImplementation
     {
         private readonly FullConnectivityHelper helper;
-        private readonly DistTest test;
+        private readonly BaseLog log;
+        private readonly FileManager fileManager;
         private ByteSize testFileSize;
 
-        public PeerDownloadTestHelpers(DistTest test)
+        public PeerDownloadTestHelpers(BaseLog log, FileManager fileManager)
         {
-            helper = new FullConnectivityHelper(test, this);
+            helper = new FullConnectivityHelper(log, this);
             testFileSize = 1.MB();
-            this.test = test;
+            this.log = log;
+            this.fileManager = fileManager;
         }
 
         public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
+        {
+            AssertFullDownloadInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess), testFileSize);
+        }
+
+        public void AssertFullDownloadInterconnectivity(IEnumerable<CodexAccess> nodes, ByteSize testFileSize)
         {
             this.testFileSize = testFileSize;
             helper.AssertFullyConnected(nodes);
@@ -33,13 +42,15 @@ namespace DistTestCore.Helpers
 
         public PeerConnectionState Check(Entry from, Entry to)
         {
+            fileManager.PushFileSet();
             var expectedFile = GenerateTestFile(from.Node, to.Node);
 
-            var contentId = from.Node.UploadFile(expectedFile);
+            using var uploadStream = File.OpenRead(expectedFile.Filename);
+            var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(uploadStream));
 
             try
             {
-                var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded");
+                var downloadedFile = Stopwatch.Measure(log, "Download", () => DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded"));
                 expectedFile.AssertIsEqual(downloadedFile);
                 return PeerConnectionState.Connection;
             }
@@ -49,16 +60,29 @@ namespace DistTestCore.Helpers
                 // We consider that as no-connection for the purpose of this test.
                 return PeerConnectionState.NoConnection;
             }
+            finally
+            {
+                fileManager.PopFileSet();
+            }
 
             // Should an exception occur during upload, then this try is inconclusive and we try again next loop.
         }
 
-        private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader)
+        private TestFile DownloadFile(CodexAccess node, string contentId, string label)
+        {
+            var downloadedFile = fileManager.CreateEmptyTestFile(label);
+            using var downloadStream = File.OpenWrite(downloadedFile.Filename);
+            using var stream = node.DownloadFile(contentId);
+            stream.CopyTo(downloadStream);
+            return downloadedFile;
+        }
+
+        private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader)
         {
             var up = uploader.GetName().Replace("<", "").Replace(">", "");
             var down = downloader.GetName().Replace("<", "").Replace(">", "");
             var label = $"~from:{up}-to:{down}~";
-            return test.GenerateTestFile(testFileSize, label);
+            return fileManager.GenerateTestFile(testFileSize, label);
         }
     }
 }

DistTestCore/Helpers/PeerFetchTestHelpers.cs

@@ -0,0 +1,96 @@
+using DistTestCore.Codex;
+using Logging;
+using NUnit.Framework;
+using static DistTestCore.Helpers.FullConnectivityHelper;
+
+namespace DistTestCore.Helpers
+{
+    public class PeerFetchTestHelpers : IFullConnectivityImplementation
+    {
+        private readonly FullConnectivityHelper helper;
+        private readonly BaseLog log;
+        private readonly IFileManager fileManager;
+        private readonly ByteSize testFileSize;
+        private readonly int expectedNumberOfBlocks;
+
+        public PeerFetchTestHelpers(BaseLog log, IFileManager fileManager)
+        {
+            helper = new FullConnectivityHelper(log, this);
+            testFileSize = 10.MB();
+            expectedNumberOfBlocks = 161;
+            this.log = log;
+            this.fileManager = fileManager;
+        }
+
+        public void AssertFullFetchInterconnectivity(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullFetchInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess));
+        }
+
+        public void AssertFullFetchInterconnectivity(IEnumerable<CodexAccess> nodes)
+        {
+            helper.AssertFullyConnected(nodes);
+        }
+
+        public string Description()
+        {
+            return "Fetch connectivity";
+        }
+
+        public string ValidateEntry(Entry entry, Entry[] allEntries)
+        {
+            return string.Empty;
+        }
+
+        public PeerConnectionState Check(Entry from, Entry to)
+        {
+            fileManager.PushFileSet();
+            var expectedFile = GenerateTestFile(from.Node, to.Node);
+
+            using var uploadStream = File.OpenRead(expectedFile.Filename);
+            var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(uploadStream));
+            var originalFetch = from.Node.DebugFetch(contentId);
+            Assert.That(Convert.ToInt32(originalFetch.numberOfBlocks), Is.EqualTo(expectedNumberOfBlocks));
+            Assert.That(originalFetch.originalBytes.Replace("'NByte", ""), Is.EqualTo("10485760"));
+
+            try
+            {
+                var fetchResponse = Stopwatch.Measure(log, "Fetch", () => to.Node.DebugFetch(contentId));
+                AssertEqual(originalFetch, fetchResponse);
+                return PeerConnectionState.Connection;
+            }
+            catch
+            {
+                return PeerConnectionState.NoConnection;
+            }
+            finally
+            {
+                fileManager.PopFileSet();
+            }
+        }
+
+        private void AssertEqual(CodexDebugFetchResponse expected, CodexDebugFetchResponse actual)
+        {
+            Assert.That(actual.originalBytes, Is.EqualTo(expected.originalBytes));
+            Assert.That(actual.blockSize, Is.EqualTo(expected.blockSize));
+            Assert.That(actual.numberOfBlocks, Is.EqualTo(expected.numberOfBlocks));
+            Assert.That(actual.version, Is.EqualTo(expected.version));
+            Assert.That(actual.hcodec, Is.EqualTo(expected.hcodec));
+            Assert.That(actual.codec, Is.EqualTo(expected.codec));
+            Assert.That(actual.@protected, Is.EqualTo(expected.@protected));
+            Assert.That(actual.blocks.Length, Is.EqualTo(expected.blocks.Length));
+            for (var i = 0; i < expected.blocks.Length; i++)
+            {
+                Assert.That(actual.blocks[i].cid, Is.EqualTo(expected.blocks[i].cid));
+            }
+        }
+
+        private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader)
+        {
+            var up = uploader.GetName().Replace("<", "").Replace(">", "");
+            var down = downloader.GetName().Replace("<", "").Replace(">", "");
+            var label = $"~from:{up}-to:{down}~";
+            return fileManager.GenerateTestFile(testFileSize, label);
+        }
+    }
+}

DistTestCore/Http.cs

@@ -1,5 +1,6 @@
 using Logging;
 using Newtonsoft.Json;
+using Serialization = Newtonsoft.Json.Serialization;
 using System.Net.Http.Headers;
 using System.Net.Http.Json;
 using Utils;
@@ -12,14 +13,21 @@ namespace DistTestCore
         private readonly ITimeSet timeSet;
         private readonly Address address;
         private readonly string baseUrl;
+        private readonly Action<HttpClient> onClientCreated;
         private readonly string? logAlias;
 
         public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
+            : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
+        {
+        }
+
+        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
         {
             this.log = log;
             this.timeSet = timeSet;
             this.address = address;
             this.baseUrl = baseUrl;
+            this.onClientCreated = onClientCreated;
             this.logAlias = logAlias;
             if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
             if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
@@ -47,23 +55,36 @@ namespace DistTestCore
 
         public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
         {
-            var json = HttpPostJson(route, body);
+            var response = PostJson(route, body);
+            var json = Time.Wait(response.Content.ReadAsStringAsync());
+            if (!response.IsSuccessStatusCode)
+            {
+                throw new HttpRequestException(json);
+            }
+            Log(GetUrl() + route, json);
             return TryJsonDeserialize<TResponse>(json);
         }
 
         public string HttpPostJson<TRequest>(string route, TRequest body)
+        {
+            var response = PostJson<TRequest>(route, body);
+            return Time.Wait(response.Content.ReadAsStringAsync());
+        }
+
+        public string HttpPostString(string route, string body)
         {
             return Retry(() =>
             {
                 using var client = GetClient();
                 var url = GetUrl() + route;
-                using var content = JsonContent.Create(body);
-                Log(url, JsonConvert.SerializeObject(body));
+                Log(url, body);
+                var content = new StringContent(body);
+                content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
                 var result = Time.Wait(client.PostAsync(url, content));
                 var str = Time.Wait(result.Content.ReadAsStringAsync());
                 Log(url, str);
                 return str;
-            }, $"HTTP-POST-JSON: {route}");
+            }, $"HTTP-POST-STRING: {route}");
         }
 
         public string HttpPostStream(string route, Stream stream)
@@ -76,7 +97,7 @@ namespace DistTestCore
                 var content = new StreamContent(stream);
                 content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
                 var response = Time.Wait(client.PostAsync(url, content));
-                var str =Time.Wait(response.Content.ReadAsStringAsync());
+                var str = Time.Wait(response.Content.ReadAsStringAsync());
                 Log(url, str);
                 return str;
             }, $"HTTP-POST-STREAM: {route}");
@@ -95,15 +116,43 @@ namespace DistTestCore
 
         public T TryJsonDeserialize<T>(string json)
         {
-            try
+            var errors = new List<string>();
+            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
             {
-                return JsonConvert.DeserializeObject<T>(json)!;
-            }
-            catch (Exception exception)
+                Error = delegate(object? sender, Serialization.ErrorEventArgs args)
+                {
+                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
+                    {
+                        errors.Add($"""
+                                    Member: '{args.ErrorContext.Member?.ToString() ?? "<null>"}'
+                                    Path: {args.ErrorContext.Path}
+                                    Error: {args.ErrorContext.Error.Message}
+                                    """);
+                        args.ErrorContext.Handled = true;
+                    }
+                }
+            });
+            if (errors.Count() > 0)
             {
-                var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}";
-                throw new InvalidOperationException(msg, exception);
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
             }
+            else if (deserialized == null)
+            {
+                throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
+            }
+            return deserialized;
+        }
+
+        private HttpResponseMessage PostJson<TRequest>(string route, TRequest body)
+        {
+            return Retry(() =>
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                using var content = JsonContent.Create(body);
+                Log(url, JsonConvert.SerializeObject(body));
+                return Time.Wait(client.PostAsync(url, content));
+            }, $"HTTP-POST-JSON: {route}");
         }
 
         private string GetUrl()
@@ -132,7 +181,12 @@ namespace DistTestCore
         {
             var client = new HttpClient();
             client.Timeout = timeSet.HttpCallTimeout();
+            onClientCreated(client);
             return client;
         }
+
+        private static void DoNothing(HttpClient client)
+        {
+        }
     }
 }

DistTestCore/Logs/DownloadedLog.cs

@@ -6,6 +6,8 @@ namespace DistTestCore.Logs
     public interface IDownloadedLog
     {
         void AssertLogContains(string expectedString);
+        string[] FindLinesThatContain(params string[] tags);
+        void DeleteFile();
     }
 
     public class DownloadedLog : IDownloadedLog
@@ -33,5 +35,30 @@ namespace DistTestCore.Logs
 
             Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
         }
+
+        public string[] FindLinesThatContain(params string[] tags)
+        {
+            var result = new List<string>();
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                if (tags.All(line.Contains))
+                {
+                    result.Add(line);
+                }
+
+                line = streamReader.ReadLine();
+            }
+
+            return result.ToArray();
+        }
+
+        public void DeleteFile()
+        {
+            File.Delete(logFile.FullFilename);
+        }
     }
 }

DistTestCore/Marketplace/CodexContractsContainerRecipe.cs

@@ -2,19 +2,15 @@
 
 namespace DistTestCore.Marketplace
 {
-    public class CodexContractsContainerRecipe : ContainerRecipeFactory
+    public class CodexContractsContainerRecipe : DefaultContainerRecipe
     {
         public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
         public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
 
-        public override string Image { get; }
+        public override string AppName => "codex-contracts";
+        public override string Image => "codexstorage/codex-contracts-eth:latest-dist-tests";
 
-        public CodexContractsContainerRecipe()
-        {
-            Image = "codexstorage/dist-tests-codex-contracts-eth:sha-b4e4897";
-        }
-
-        protected override void Initialize(StartupConfig startupConfig)
+        protected override void InitializeRecipe(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -6,8 +6,8 @@ namespace DistTestCore.Marketplace
     public class CodexContractsStarter : BaseStarter
     {
 
-        public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public CodexContractsStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
@@ -15,7 +15,7 @@ namespace DistTestCore.Marketplace
         {
             LogStart("Deploying Codex Marketplace...");
 
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]);
 
             var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig);
@@ -25,7 +25,7 @@ namespace DistTestCore.Marketplace
             WaitUntil(() =>
             {
                 var logHandler = new ContractsReadyLogHandler(Debug);
-                workflow.DownloadContainerLog(container, logHandler);
+                workflow.DownloadContainerLog(container, logHandler, null);
                 return logHandler.Found;
             });
             Log("Contracts deployed. Extracting addresses...");
@@ -91,8 +91,13 @@ namespace DistTestCore.Marketplace
         {
             debug(line);
             if (line.Contains(RequiredCompiledString)) SeenCompileString = true;
+            if (line.Contains(ReadyString))
+            {
+                if (!SeenCompileString) throw new Exception("CodexContracts deployment failed. " +
+                    "Solidity files not compiled before process exited.");
 
-            if (SeenCompileString && line.Contains(ReadyString)) Found = true;
+                Found = true;
+            }
         }
     }
 }

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -80,7 +80,7 @@ namespace DistTestCore.Marketplace
         private string FetchPubKey()
         {
             var enodeFinder = new PubKeyFinder(s => log.Debug(s));
-            workflow.DownloadContainerLog(container, enodeFinder);
+            workflow.DownloadContainerLog(container, enodeFinder, null);
             return enodeFinder.GetPubKey();
         }
 

DistTestCore/Marketplace/GethBootstrapNodeStarter.cs

@@ -4,8 +4,8 @@ namespace DistTestCore.Marketplace
 {
     public class GethBootstrapNodeStarter : BaseStarter
     {
-        public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public GethBootstrapNodeStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
@@ -14,7 +14,7 @@ namespace DistTestCore.Marketplace
             LogStart("Starting Geth bootstrap node...");
             var startupConfig = CreateBootstrapStartupConfig();
             
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
             var bootstrapContainer = containers.Containers[0];

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -7,8 +7,8 @@ namespace DistTestCore.Marketplace
     {
         private int companionAccountIndex = 0;
 
-        public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public GethCompanionNodeStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
@@ -18,7 +18,7 @@ namespace DistTestCore.Marketplace
 
             var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
 
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
             var container = containers.Containers[0];

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -2,7 +2,7 @@
 
 namespace DistTestCore.Marketplace
 {
-    public class GethContainerRecipe : ContainerRecipeFactory
+    public class GethContainerRecipe : DefaultContainerRecipe
     {
         private const string defaultArgs = "--ipcdisable --syncmode full";
 
@@ -10,14 +10,10 @@ namespace DistTestCore.Marketplace
         public const string DiscoveryPortTag = "disc_port";
         public const string AccountsFilename = "accounts.csv";
 
-        public override string Image { get; }
+        public override string AppName => "geth";
+        public override string Image => "codexstorage/dist-tests-geth:latest";
 
-        public GethContainerRecipe()
-        {
-            Image = "codexstorage/dist-tests-geth:sha-b788a2d";
-        }
-
-        protected override void Initialize(StartupConfig startupConfig)
+        protected override void InitializeRecipe(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<GethStartupConfig>();
 

DistTestCore/Metrics/GrafanaContainerRecipe.cs

@@ -0,0 +1,25 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Metrics
+{
+    public class GrafanaContainerRecipe : DefaultContainerRecipe
+    {
+        public override string AppName => "grafana";
+        public override string Image => "grafana/grafana-oss:10.0.3";
+
+        public const string DefaultAdminUser = "adminium";
+        public const string DefaultAdminPassword = "passwordium";
+
+        protected override void InitializeRecipe(StartupConfig startupConfig)
+        {
+            AddExposedPort(3000);
+
+            AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true");
+            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org.");
+            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor");
+
+            AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser);
+            AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword);
+        }
+    }
+}

DistTestCore/Metrics/MetricsMode.cs

@@ -0,0 +1,9 @@
+namespace DistTestCore.Metrics
+{
+    public enum MetricsMode
+    {
+        None,
+        Record,
+        Dashboard
+    }
+}

DistTestCore/Metrics/PrometheusContainerRecipe.cs

@@ -2,16 +2,12 @@
 
 namespace DistTestCore.Metrics
 {
-    public class PrometheusContainerRecipe : ContainerRecipeFactory
+    public class PrometheusContainerRecipe : DefaultContainerRecipe
     {
-        public override string Image { get; }
+        public override string AppName => "prometheus";
+        public override string Image => "codexstorage/dist-tests-prometheus:latest";
 
-        public PrometheusContainerRecipe()
-        {
-            Image = "codexstorage/dist-tests-prometheus:sha-f97d7fd";
-        }
-
-        protected override void Initialize(StartupConfig startupConfig)
+        protected override void InitializeRecipe(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<PrometheusStartupConfig>();
 

DistTestCore/OnlineCodexNode.cs

@@ -4,6 +4,7 @@ using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
 using Logging;
 using NUnit.Framework;
+using Utils;
 
 namespace DistTestCore
 {
@@ -15,7 +16,7 @@ namespace DistTestCore
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(IOnlineCodexNode node);
-        IDownloadedLog DownloadLog();
+        IDownloadedLog DownloadLog(int? tailLines = null);
         IMetricsAccess Metrics { get; }
         IMarketplaceAccess Marketplace { get; }
         CodexDebugVersionResponse Version { get; }
@@ -67,6 +68,7 @@ namespace DistTestCore
             using var fileStream = File.OpenRead(file.Filename);
 
             var logMessage = $"Uploading file {file.Describe()}...";
+            Log(logMessage);
             var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
             {
                 return CodexAccess.UploadFile(fileStream);
@@ -82,6 +84,7 @@ namespace DistTestCore
         public TestFile? DownloadContent(ContentId contentId, string fileLabel = "")
         {
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
+            Log(logMessage);
             var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel);
             Stopwatch.Measure(lifecycle.Log, logMessage, () => DownloadToFile(contentId.Id, file));
             Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
@@ -100,9 +103,9 @@ namespace DistTestCore
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public IDownloadedLog DownloadLog()
+        public IDownloadedLog DownloadLog(int? tailLines = null)
         {
-            return lifecycle.DownloadLog(CodexAccess.Container);
+            return lifecycle.DownloadLog(CodexAccess.Container, tailLines);
         }
 
         public ICodexSetup BringOffline()
@@ -116,7 +119,7 @@ namespace DistTestCore
 
         public void EnsureOnlineGetVersionResponse()
         {
-            var debugInfo = CodexAccess.GetDebugInfo();
+            var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
             var nodePeerId = debugInfo.id;
             var nodeName = CodexAccess.Container.Name;
 
@@ -135,14 +138,9 @@ namespace DistTestCore
             var multiAddress = peerInfo.addrs.First();
             // Todo: Is there a case where First address in list is not the way?
 
-            if (Group == peer.Group)
-            {
-                return multiAddress;
-            }
-
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip);
+            return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip);
         }
 
         private void DownloadToFile(string contentId, TestFile file)

DistTestCore/PrometheusStarter.cs

@@ -7,23 +7,21 @@ namespace DistTestCore
 {
     public class PrometheusStarter : BaseStarter
     {
-        public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public PrometheusStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
-        public RunningContainers CollectMetricsFor(RunningContainers containers)
+        public RunningContainers CollectMetricsFor(RunningContainers[] containers)
         {
             LogStart($"Starting metrics server for {containers.Describe()}");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers)));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers())));
 
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 
-            LogEnd("Metrics server started.");
-
             return runningContainers;
         }
 
@@ -31,7 +29,7 @@ namespace DistTestCore
         {
             var config = "";
             config += "global:\n";
-            config += "  scrape_interval: 30s\n";
+            config += "  scrape_interval: 10s\n";
             config += "  scrape_timeout: 10s\n";
             config += "\n";
             config += "scrape_configs:\n";

DistTestCore/TestLifecycle.cs

@@ -1,5 +1,7 @@
 using DistTestCore.Codex;
 using DistTestCore.Logs;
+using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
 using KubernetesWorkflow;
 using Logging;
 using Utils;
@@ -10,21 +12,19 @@ namespace DistTestCore
     {
         private readonly DateTime testStart;
 
-        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet)
-            : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)))
-        {
-        }
-
-        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
+        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
         {
             Log = log;
             Configuration = configuration;
             TimeSet = timeSet;
 
+            WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), testNamespace);
+
             FileManager = new FileManager(Log, configuration);
-            CodexStarter = new CodexStarter(this, workflowCreator);
-            PrometheusStarter = new PrometheusStarter(this, workflowCreator);
-            GethStarter = new GethStarter(this, workflowCreator);
+            CodexStarter = new CodexStarter(this);
+            PrometheusStarter = new PrometheusStarter(this);
+            GrafanaStarter = new GrafanaStarter(this);
+            GethStarter = new GethStarter(this);
             testStart = DateTime.UtcNow;
             CodexVersion = null;
 
@@ -34,9 +34,11 @@ namespace DistTestCore
         public BaseLog Log { get; }
         public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
+        public WorkflowCreator WorkflowCreator { get; }
         public FileManager FileManager { get; }
         public CodexStarter CodexStarter { get; }
         public PrometheusStarter PrometheusStarter { get; }
+        public GrafanaStarter GrafanaStarter { get; }
         public GethStarter GethStarter { get; }
         public CodexDebugVersionResponse? CodexVersion { get; private set; }
 
@@ -46,14 +48,14 @@ namespace DistTestCore
             FileManager.DeleteAllTestFiles();
         }
 
-        public IDownloadedLog DownloadLog(RunningContainer container)
+        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
         {
             var subFile = Log.CreateSubfile();
             var description = container.Name;
             var handler = new LogDownloadHandler(container, description, subFile);
 
             Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-            CodexStarter.DownloadLog(container, handler);
+            CodexStarter.DownloadLog(container, handler, tailLines);
 
             return new DownloadedLog(subFile, description);
         }
@@ -68,5 +70,24 @@ namespace DistTestCore
         {
             if (CodexVersion == null) CodexVersion = version;
         }
+
+        public ApplicationIds GetApplicationIds()
+        {
+            return new ApplicationIds(
+                codexId: GetCodexId(),
+                gethId: new GethContainerRecipe().Image,
+                prometheusId: new PrometheusContainerRecipe().Image,
+                codexContractsId: new CodexContractsContainerRecipe().Image,
+                grafanaId: new GrafanaContainerRecipe().Image
+            );
+        }
+
+        private string GetCodexId()
+        {
+            var v = CodexVersion;
+            if (v == null) return new CodexContainerRecipe().Image;
+            if (v.version != "untagged build") return v.version;
+            return v.revision;
+        }
     }
 }

DistTestCore/Timing.cs

@@ -21,7 +21,7 @@ namespace DistTestCore
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromSeconds(10);
+            return TimeSpan.FromMinutes(5);
         }
 
         public TimeSpan HttpCallRetryTime()
@@ -36,12 +36,12 @@ namespace DistTestCore
 
         public TimeSpan WaitForK8sServiceDelay()
         {
-            return TimeSpan.FromSeconds(1);
+            return TimeSpan.FromSeconds(10);
         }
 
         public TimeSpan K8sOperationTimeout()
         {
-            return TimeSpan.FromMinutes(1);
+            return TimeSpan.FromMinutes(30);
         }
 
         public TimeSpan WaitForMetricTimeout()

KubernetesWorkflow/CommandRunner.cs

@@ -1,5 +1,4 @@
-using k8s;
-using Utils;
+using Utils;
 
 namespace KubernetesWorkflow
 {

KubernetesWorkflow/ContainerRecipe.cs

@@ -2,13 +2,15 @@
 {
     public class ContainerRecipe
     {
-        public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, object[] additionals)
+        public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, object[] additionals)
         {
             Number = number;
             Image = image;
             ExposedPorts = exposedPorts;
             InternalPorts = internalPorts;
             EnvVars = envVars;
+            PodLabels = podLabels;
+            PodAnnotations = podAnnotations;
             Additionals = additionals;
         }
 
@@ -18,6 +20,8 @@
         public Port[] ExposedPorts { get; }
         public Port[] InternalPorts { get; }
         public EnvVar[] EnvVars { get; }
+        public PodLabels PodLabels { get; }
+        public PodAnnotations PodAnnotations { get; }
         public object[] Additionals { get; }
 
         public Port GetPortByTag(string tag)

KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -5,6 +5,8 @@
         private readonly List<Port> exposedPorts = new List<Port>();
         private readonly List<Port> internalPorts = new List<Port>();
         private readonly List<EnvVar> envVars = new List<EnvVar>();
+        private readonly PodLabels podLabels = new PodLabels();
+        private readonly PodAnnotations podAnnotations = new PodAnnotations();
         private readonly List<object> additionals = new List<object>();
         private RecipeComponentFactory factory = null!;
 
@@ -16,17 +18,26 @@
 
             Initialize(config);
 
-            var recipe = new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray(), additionals.ToArray());
+            var recipe = new ContainerRecipe(containerNumber, Image,
+                exposedPorts.ToArray(),
+                internalPorts.ToArray(), 
+                envVars.ToArray(), 
+                podLabels.Clone(),
+                podAnnotations.Clone(),
+                additionals.ToArray());
 
             exposedPorts.Clear();
             internalPorts.Clear();
             envVars.Clear();
+            podLabels.Clear();
+            podAnnotations.Clear();
             additionals.Clear();
             this.factory = null!;
 
             return recipe;
         }
 
+        public abstract string AppName { get; }
         public abstract string Image { get; }
         protected int ContainerNumber { get; private set; } = 0;
         protected int Index { get; private set; } = 0;
@@ -39,6 +50,13 @@
             return p;
         }
 
+        protected Port AddExposedPort(int number, string tag = "")
+        {
+            var p = factory.CreatePort(number, tag);
+            exposedPorts.Add(p);
+            return p;
+        }
+
         protected Port AddInternalPort(string tag = "")
         {
             var p = factory.CreatePort(tag);
@@ -66,6 +84,16 @@
             envVars.Add(factory.CreateEnvVar(name, value.Number));
         }
 
+        protected void AddPodLabel(string name, string value)
+        {
+            podLabels.Add(name, value);
+        }
+
+        protected void AddPodAnnotation(string name, string value)
+        {
+            podAnnotations.Add(name, value);
+        }
+
         protected void Additional(object userData)
         {
             additionals.Add(userData);

KubernetesWorkflow/CrashWatcher.cs

@@ -0,0 +1,95 @@
+using k8s;
+using Logging;
+
+namespace KubernetesWorkflow
+{
+    public class CrashWatcher
+    {
+        private readonly BaseLog log;
+        private readonly KubernetesClientConfiguration config;
+        private readonly string k8sNamespace;
+        private readonly RunningContainer container;
+        private ILogHandler? logHandler;
+        private CancellationTokenSource cts;
+        private Task? worker;
+        private Exception? workerException;
+
+        public CrashWatcher(BaseLog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container)
+        {
+            this.log = log;
+            this.config = config;
+            this.k8sNamespace = k8sNamespace;
+            this.container = container;
+            cts = new CancellationTokenSource();
+        }
+
+        public void Start(ILogHandler logHandler)
+        {
+            if (worker != null) throw new InvalidOperationException();
+
+            this.logHandler = logHandler;
+            cts = new CancellationTokenSource();
+            worker = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            if (worker == null) throw new InvalidOperationException();
+
+            cts.Cancel();
+            worker.Wait();
+            worker = null;
+
+            if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
+        }
+
+        public bool HasContainerCrashed()
+        {
+            using var client = new Kubernetes(config);
+            return HasContainerBeenRestarted(client, container.Pod.PodInfo.Name);
+        }
+
+        private void Worker()
+        {
+            try
+            {
+                MonitorContainer(cts.Token);
+            }
+            catch (Exception ex)
+            {
+                workerException = ex;
+            }
+        }
+
+        private void MonitorContainer(CancellationToken token)
+        {
+            using var client = new Kubernetes(config);
+            while (!token.IsCancellationRequested)
+            {
+                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(1));
+
+                var pod = container.Pod;
+                var recipe = container.Recipe;
+                var podName = pod.PodInfo.Name;
+                if (HasContainerBeenRestarted(client, podName))
+                {
+                    DownloadCrashedContainerLogs(client, podName, recipe);
+                    return;
+                }
+            }
+        }
+
+        private bool HasContainerBeenRestarted(Kubernetes client, string podName)
+        {
+            var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
+            return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+        }
+
+        private void DownloadCrashedContainerLogs(Kubernetes client, string podName, ContainerRecipe recipe)
+        {
+            log.Log("Pod crash detected for " + container.Name);
+            using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipe.Name, previous: true);
+            logHandler!.Log(stream);
+        }
+    }
+}

KubernetesWorkflow/K8sCluster.cs

@@ -8,7 +8,7 @@ namespace KubernetesWorkflow
         {
             Configuration = configuration;
         }
-       
+
         public Configuration Configuration { get; }
         public string HostAddress { get; private set; } = string.Empty;
         public K8sNodeLabel[] AvailableK8sNodes { get; set; } = new K8sNodeLabel[0];

KubernetesWorkflow/K8sController.cs

@@ -51,10 +51,10 @@ namespace KubernetesWorkflow
             WaitUntilPodOffline(pod.PodInfo.Name);
         }
 
-        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
+        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler, int? tailLines)
         {
             log.Debug();
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, tailLines: tailLines));
             logHandler.Log(stream);
         }
 
@@ -219,6 +219,19 @@ namespace KubernetesWorkflow
                                         }
                                     }
                                 }
+                            },
+                            new V1NetworkPolicyIngressRule
+                            {
+                                FromProperty = new List<V1NetworkPolicyPeer>
+                                {
+                                    new V1NetworkPolicyPeer
+                                    {
+                                        NamespaceSelector = new V1LabelSelector
+                                        {
+                                            MatchLabels = GetPrometheusNamespaceSelector()
+                                        }
+                                    }
+                                }
                             }
                         },
                         Egress = new List<V1NetworkPolicyEgressRule>
@@ -272,7 +285,7 @@ namespace KubernetesWorkflow
                                     {
                                         IpBlock = new V1IPBlock
                                         {
-                                          Cidr = "0.0.0.0/0"
+                                            Cidr = "0.0.0.0/0"
                                         }
                                     }
                                 },
@@ -314,19 +327,20 @@ namespace KubernetesWorkflow
             var deploymentSpec = new V1Deployment
             {
                 ApiVersion = "apps/v1",
-                Metadata = CreateDeploymentMetadata(),
+                Metadata = CreateDeploymentMetadata(containerRecipes),
                 Spec = new V1DeploymentSpec
                 {
                     Replicas = 1,
                     Selector = new V1LabelSelector
                     {
-                        MatchLabels = GetSelector()
+                        MatchLabels = GetSelector(containerRecipes)
                     },
                     Template = new V1PodTemplateSpec
                     {
                         Metadata = new V1ObjectMeta
                         {
-                            Labels = GetSelector()
+                            Labels = GetSelector(containerRecipes),
+                            Annotations = GetAnnotations(containerRecipes)
                         },
                         Spec = new V1PodSpec
                         {
@@ -360,9 +374,9 @@ namespace KubernetesWorkflow
             };
         }
 
-        private IDictionary<string, string> GetSelector()
+        private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes)
         {
-            return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
+            return containerRecipes.First().PodLabels.GetLabels();
         }
 
         private IDictionary<string, string> GetRunnerNamespaceSelector()
@@ -370,13 +384,24 @@ namespace KubernetesWorkflow
             return new Dictionary<string, string> { { "kubernetes.io/metadata.name", "default" } };
         }
 
-        private V1ObjectMeta CreateDeploymentMetadata()
+        private IDictionary<string, string> GetPrometheusNamespaceSelector()
+        {
+            return new Dictionary<string, string> { { "kubernetes.io/metadata.name", "monitoring" } };
+        }
+
+        private IDictionary<string, string> GetAnnotations(ContainerRecipe[] containerRecipes)
+        {
+            return containerRecipes.First().PodAnnotations.GetAnnotations();
+        }
+
+        private V1ObjectMeta CreateDeploymentMetadata(ContainerRecipe[] containerRecipes)
         {
             return new V1ObjectMeta
             {
                 Name = "deploy-" + workflowNumberSource.WorkflowNumber,
                 NamespaceProperty = K8sTestNamespace,
-                Labels = GetSelector()
+                Labels = GetSelector(containerRecipes),
+                Annotations = GetAnnotations(containerRecipes)
             };
         }
 
@@ -423,7 +448,7 @@ namespace KubernetesWorkflow
             return new V1ContainerPort
             {
                 Name = GetNameForPort(recipe, port),
-                ContainerPort = port.Number 
+                ContainerPort = port.Number
             };
         }
 
@@ -456,7 +481,7 @@ namespace KubernetesWorkflow
                 Spec = new V1ServiceSpec
                 {
                     Type = "NodePort",
-                    Selector = GetSelector(),
+                    Selector = GetSelector(containerRecipes),
                     Ports = ports
                 }
             };
@@ -602,6 +627,11 @@ namespace KubernetesWorkflow
 
         #endregion
 
+        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        {
+            return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sTestNamespace, container);
+        }
+
         private PodInfo FetchNewPod()
         {
             var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;

KubernetesWorkflow/PodAnnotations.cs

@@ -0,0 +1,29 @@
+namespace KubernetesWorkflow
+{
+    public class PodAnnotations
+    {
+        private readonly Dictionary<string, string> annotations = new Dictionary<string, string>();
+
+        public void Add(string key, string value)
+        {
+            annotations.Add(key, value);
+        }
+
+        public PodAnnotations Clone()
+        {
+            var result = new PodAnnotations();
+            foreach (var entry in annotations) result.Add(entry.Key, entry.Value);
+            return result;
+        }
+
+        public void Clear()
+        {
+            annotations.Clear();
+        }
+
+        internal Dictionary<string, string> GetAnnotations()
+        {
+            return annotations;
+        }
+    }
+}

KubernetesWorkflow/PodLabels.cs

@@ -0,0 +1,42 @@
+namespace KubernetesWorkflow
+{
+    public class PodLabels
+    {
+        private readonly Dictionary<string, string> labels = new Dictionary<string, string>();
+
+        public void Add(string key, string value)
+        {
+            labels.Add(key, Format(value));
+        }
+
+        public PodLabels Clone()
+        {
+            var result = new PodLabels();
+            foreach (var entry in labels) result.Add(entry.Key, entry.Value);
+            return result;
+        }
+
+        public void Clear()
+        {
+            labels.Clear();
+        }
+
+        private static string Format(string s)
+        {
+            var result = s.ToLowerInvariant()
+                .Replace(":", "-")
+                .Replace("/", "-")
+                .Replace("\\", "-")
+                .Replace("[", "-")
+                .Replace("]", "-")
+                .Replace(",", "-");
+
+            return result.Trim('-');
+        }
+
+        internal Dictionary<string, string> GetLabels()
+        {
+            return labels;
+        }
+    }
+}

KubernetesWorkflow/RecipeComponentFactory.cs

@@ -7,6 +7,11 @@ namespace KubernetesWorkflow
     {
         private NumberSource portNumberSource = new NumberSource(8080);
 
+        public Port CreatePort(int number, string tag)
+        {
+            return new Port(number, tag);
+        }
+
         public Port CreatePort(string tag)
         {
             return new Port(portNumberSource.GetNextNumber(), tag);

KubernetesWorkflow/RunningContainers.cs

@@ -1,4 +1,5 @@
-using Utils;
+using Newtonsoft.Json;
+using Utils;
 
 namespace KubernetesWorkflow
 {
@@ -39,5 +40,21 @@ namespace KubernetesWorkflow
         public Port[] ServicePorts { get; }
         public Address ClusterExternalAddress { get; }
         public Address ClusterInternalAddress { get; }
+
+        [JsonIgnore]
+        public CrashWatcher? CrashWatcher { get; set; }
+    }
+
+    public static class RunningContainersExtensions
+    {
+        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
+        {
+            return runningContainers.SelectMany(c => c.Containers).ToArray();
+        }
+
+        public static string Describe(this RunningContainers[] runningContainers)
+        {
+            return string.Join(",", runningContainers.Select(c => c.Describe()));
+        }
     }
 }

KubernetesWorkflow/StartupWorkflow.cs

@@ -33,6 +33,11 @@ namespace KubernetesWorkflow
             });
         }
 
+        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        {
+            return K8s(controller => controller.CreateCrashWatcher(container));
+        }
+
         public void Stop(RunningContainers runningContainers)
         {
             K8s(controller =>
@@ -41,11 +46,11 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler)
+        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler);
+                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler, tailLines);
             });
         }
 

KubernetesWorkflow/WorkflowCreator.cs

@@ -12,16 +12,11 @@ namespace KubernetesWorkflow
         private readonly BaseLog log;
         private readonly string testNamespace;
 
-        public WorkflowCreator(BaseLog log, Configuration configuration)
-            : this(log, configuration, Guid.NewGuid().ToString().ToLowerInvariant())
-        {
-        }
-
-        public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespacePostfix)
+        public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespace)
         {
             cluster = new K8sCluster(configuration);
             this.log = log;
-            testNamespace = testNamespacePostfix;
+            this.testNamespace = testNamespace.ToLowerInvariant();
         }
 
         public StartupWorkflow CreateWorkflow()

Logging/ApplicationIds.cs

@@ -0,0 +1,20 @@
+namespace Logging
+{
+    public class ApplicationIds
+    {
+        public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId, string grafanaId)
+        {
+            CodexId = codexId;
+            GethId = gethId;
+            PrometheusId = prometheusId;
+            CodexContractsId = codexContractsId;
+            GrafanaId = grafanaId;
+        }
+
+        public string CodexId { get; }
+        public string GethId { get; }
+        public string PrometheusId { get; }
+        public string CodexContractsId { get; }
+        public string GrafanaId { get; }
+    }
+}

Logging/BaseLog.cs

@@ -60,6 +60,7 @@ namespace Logging
         public virtual void AddStringReplace(string from, string to)
         {
             if (string.IsNullOrWhiteSpace(from)) return;
+            if (replacements.Any(r => r.From == from)) return;
             replacements.Add(new BaseLogStringReplacement(from, to));
         }
 
@@ -98,20 +99,20 @@ namespace Logging
 
     public class BaseLogStringReplacement
     {
-        private readonly string from;
-        private readonly string to;
-
         public BaseLogStringReplacement(string from, string to)
         {
-            this.from = from;
-            this.to = to;
+            From = from;
+            To = to;
 
             if (string.IsNullOrEmpty(from) || string.IsNullOrEmpty(to) || from == to) throw new ArgumentException();
         }
 
+        public string From { get; }
+        public string To { get; }
+
         public string Apply(string msg)
         {
-            return msg.Replace(from, to);
+            return msg.Replace(From, To);
         }
     }
 }

Logging/NameUtils.cs

@@ -22,6 +22,7 @@ namespace Logging
         public static string GetRawFixtureName()
         {
             var test = TestContext.CurrentContext.Test;
+            if (test.ClassName!.Contains("AdhocContext")) return "none";
             var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
             return className.Replace('.', '-');
         }
@@ -29,6 +30,7 @@ namespace Logging
         public static string GetCategoryName()
         {
             var test = TestContext.CurrentContext.Test;
+            if (test.ClassName!.Contains("AdhocContext")) return "none";
             return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.'));
         }
 
@@ -45,7 +47,7 @@ namespace Logging
         private static string GetEnvVar(string name)
         {
             var v = Environment.GetEnvironmentVariable(name);
-            if (string.IsNullOrEmpty(v)) return $"EnvVar'{name}'NotSet";
+            if (string.IsNullOrEmpty(v)) return $"EnvVar-{name}-NotSet";
             return v;
         }
 

Logging/NullLog.cs

@@ -6,9 +6,11 @@
         {
         }
 
+        public string FullFilename { get; set; } = "NULL";
+
         protected override string GetFullName()
         {
-            return "NULL";
+            return FullFilename;
         }
 
         public override void Log(string message)

Logging/StatusLog.cs

@@ -14,7 +14,7 @@ namespace Logging
             fixtureName = NameUtils.GetRawFixtureName();
         }
 
-        public void ConcludeTest(string resultStatus, string testDuration, string codexId)
+        public void ConcludeTest(string resultStatus, string testDuration, ApplicationIds applicationIds)
         {
             Write(new StatusLogJson
             {
@@ -22,7 +22,11 @@ namespace Logging
                 runid = NameUtils.GetRunId(),
                 status = resultStatus,
                 testid = NameUtils.GetTestId(),
-                codexid = codexId,
+                codexid = applicationIds.CodexId,
+                gethid = applicationIds.GethId,
+                prometheusid = applicationIds.PrometheusId,
+                codexcontractsid = applicationIds.CodexContractsId,
+                grafanaid = applicationIds.GrafanaId,
                 category = NameUtils.GetCategoryName(),
                 fixturename = fixtureName,
                 testname = NameUtils.GetTestMethodName(),
@@ -53,6 +57,10 @@ namespace Logging
         public string status { get; set; } = string.Empty;
         public string testid { get; set; } = string.Empty; 
         public string codexid { get; set; } = string.Empty;
+        public string gethid { get; set; } = string.Empty;
+        public string prometheusid { get; set; } = string.Empty;
+        public string codexcontractsid { get; set; } = string.Empty;
+        public string grafanaid { get; set; } = string.Empty;
         public string category { get; set; } = string.Empty;
         public string fixturename { get; set; } = string.Empty;
         public string testname { get; set; } = string.Empty;

LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs

@@ -1,5 +1,4 @@
-using DistTestCore.Helpers;
-using DistTestCore;
+using DistTestCore;
 using NUnit.Framework;
 
 namespace TestsLong.DownloadConnectivityTests
@@ -16,7 +15,7 @@ namespace TestsLong.DownloadConnectivityTests
         {
             for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
 
-            PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
+            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
         }
     }
 }

README.md

@@ -3,7 +3,6 @@
 
 Using a common dotnet unit-test framework and a few other libraries, this project allows you to write tests that use multiple Codex node instances in various configurations to test the distributed system in a controlled, reproducible environment.
 
-
 Nim-Codex: https://github.com/codex-storage/nim-codex  
 Dotnet: v6.0  
 Kubernetes: v1.25.4  
@@ -23,7 +22,7 @@ Test executing can be configured using the following environment variables.
 |----------------|------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|
 | KUBECONFIG     | Optional path (abs or rel) to kubeconfig YAML file. When null, uses system default (docker-desktop) kubeconfig if available.                         | (null)              |
 | LOGPATH        | Path (abs or rel) where log files will be saved.                                                                                                     | "CodexTestLogs"     |
-| LOGDEBUG       | When "true", enables additional test-runner debug log output.                                                                                                    | "false"             |
+| LOGDEBUG       | When "true", enables additional test-runner debug log output.                                                                                        | "false"             |
 | DATAFILEPATH   | Path (abs or rel) where temporary test data files will be saved.                                                                                     | "TestDataFiles"     |
 | LOGLEVEL       | Codex log-level. (case-insensitive)                                                                                                                  | "Trace"             |
 | RUNNERLOCATION | Use "ExternalToCluster" when test app is running outside of the k8s cluster. Use "InternalToCluster" when tests are run from inside a pod/container. | "ExternalToCluster" |
@@ -32,10 +31,10 @@ Test executing can be configured using the following environment variables.
 Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs.
 
 ## How to contribute tests
-An important goal of the test infra is to provide a simple, accessible way for developers to write their tests. If you want to contribute tests for Codex, please follow the steps [HERE](/CONTRIBUTINGTESTS.MD).
+An important goal of the test infra is to provide a simple, accessible way for developers to write their tests. If you want to contribute tests for Codex, please follow the steps [HERE](/CONTRIBUTINGTESTS.md).
 
 ## Run the tests on your machine
-Creating tests is much easier when you can debug them on your local system. This is possible, but requires some set-up. If you want to be able to run the tests on your local system, follow the steps [HERE](/LOCALSETUP.MD). Please note that tests which require explicit node locations cannot be executed locally. (Well, you could comment out the location statements and then it would probably work. But that might impact the validity/usefulness of the test.)
+Creating tests is much easier when you can debug them on your local system. This is possible, but requires some set-up. If you want to be able to run the tests on your local system, follow the steps [HERE](/docs/LOCALSETUP.md). Please note that tests which require explicit node locations cannot be executed locally. (Well, you could comment out the location statements and then it would probably work. But that might impact the validity/usefulness of the test.)
 
 ## Missing functionality
 Surely the test-infra doesn't do everything we'll need it to do. If you're running into a limitation and would like to request a new feature for the test-infra, please create an issue.

Tests/BasicTests/ContinuousSubstitute.cs

@@ -0,0 +1,252 @@
+using DistTestCore;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.BasicTests
+{
+    [Ignore("Used for debugging continuous tests")]
+    [TestFixture]
+    public class ContinuousSubstitute : AutoBootstrapDistTest
+    {
+        [Test]
+        public void ContinuousTestSubstitute()
+        {
+            var group = SetupCodexNodes(5, o => o
+                    .EnableMetrics()
+                    .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
+                    .WithBlockTTL(TimeSpan.FromMinutes(2))
+                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
+                    .WithBlockMaintenanceNumber(10000)
+                    .WithBlockTTL(TimeSpan.FromMinutes(2))
+                    .WithStorageQuota(1.GB()));
+
+            var nodes = group.Cast<OnlineCodexNode>().ToArray();
+
+            foreach (var node in nodes)
+            {
+                node.Marketplace.MakeStorageAvailable(
+                size: 500.MB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 1024.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(5));
+            }
+
+            var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
+            while (DateTime.UtcNow < endTime)
+            {
+                var allNodes = nodes.ToList();
+                var primary = allNodes.PickOneRandom();
+                var secondary = allNodes.PickOneRandom();
+
+                Log("Run Test");
+                PerformTest(primary, secondary);
+
+                Thread.Sleep(TimeSpan.FromSeconds(5));
+            }
+        }
+
+        [Test]
+        public void PeerTest()
+        {
+            var group = SetupCodexNodes(5, o => o
+                    .EnableMetrics()
+                    .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
+                    .WithBlockTTL(TimeSpan.FromMinutes(2))
+                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
+                    .WithBlockMaintenanceNumber(10000)
+                    .WithBlockTTL(TimeSpan.FromMinutes(2))
+                    .WithStorageQuota(1.GB()));
+
+            var nodes = group.Cast<OnlineCodexNode>().ToArray();
+
+            var checkTime = DateTime.UtcNow + TimeSpan.FromMinutes(1);
+            var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
+            while (DateTime.UtcNow < endTime)
+            {
+                CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
+                CheckRoutingTables(GetAllOnlineCodexNodes());
+
+                var node = RandomUtils.PickOneRandom(nodes.ToList());
+                var file = GenerateTestFile(50.MB());
+                node.UploadFile(file);
+
+                Thread.Sleep(20000);
+            }
+        }
+
+        private void CheckRoutingTables(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            var all = nodes.ToArray();
+            var allIds = all.Select(n => n.GetDebugInfo().table.localNode.nodeId).ToArray();
+
+            var errors = all.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray();
+            
+            if (errors.Any())
+            {
+                Assert.Fail(string.Join(Environment.NewLine, errors));
+            }
+        }
+
+        private string AreAllPresent(IOnlineCodexNode n, string[] allIds)
+        {
+            var info = n.GetDebugInfo();
+            var known = info.table.nodes.Select(n => n.nodeId).ToArray();
+            var expected = allIds.Where(i => i != info.table.localNode.nodeId).ToArray();
+
+            if (!expected.All(ex => known.Contains(ex)))
+            {
+                return $"Not all of '{string.Join(",", expected)}' were present in routing table: '{string.Join(",", known)}'";
+            }
+
+            return string.Empty;
+        }
+
+        private ByteSize fileSize = 80.MB();
+
+        private void PerformTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
+        {
+            ScopedTestFiles(() =>
+            {
+                var testFile = GenerateTestFile(fileSize);
+
+                var contentId = primary.UploadFile(testFile);
+
+                var downloadedFile = secondary.DownloadContent(contentId);
+
+                testFile.AssertIsEqual(downloadedFile);
+            });
+        }
+        
+        [Test]
+        public void HoldMyBeerTest()
+        {
+            var blockExpirationTime = TimeSpan.FromMinutes(3);
+            var group = SetupCodexNodes(3, o => o
+                    .EnableMetrics()
+                    .WithBlockTTL(blockExpirationTime)
+                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
+                    .WithBlockMaintenanceNumber(10000)
+                    .WithStorageQuota(2000.MB()));
+
+            var nodes = group.Cast<OnlineCodexNode>().ToArray();
+
+            var endTime = DateTime.UtcNow + TimeSpan.FromHours(24);
+
+            var filesize = 80.MB();
+            double codexDefaultBlockSize = 31 * 64 * 33;
+            var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
+            var sizeInBytes = filesize.SizeInBytes;
+            Assert.That(numberOfBlocks, Is.EqualTo(1282));
+
+            var startTime = DateTime.UtcNow;
+            var successfulUploads = 0;
+            var successfulDownloads = 0;
+
+            while (DateTime.UtcNow < endTime)
+            {
+                foreach (var node in nodes)
+                {
+                    try
+                    {
+                        Thread.Sleep(TimeSpan.FromSeconds(5));
+
+                        ScopedTestFiles(() =>
+                        {
+                            var uploadStartTime = DateTime.UtcNow;
+                            var file = GenerateTestFile(filesize);
+                            var cid = node.UploadFile(file);
+
+                            var cidTag = cid.Id.Substring(cid.Id.Length - 6);
+                            Measure("upload-log-asserts", () =>
+                            {
+                                var uploadLog = node.DownloadLog(tailLines: 50000);
+
+                                var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
+                                uploadLog.DeleteFile();
+
+                                var storeLine = GetLineForCidTag(storeLines, cidTag);
+                                AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
+                            });
+                            successfulUploads++;
+
+                            var uploadTimeTaken = DateTime.UtcNow - uploadStartTime;
+                            if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10)))
+                            {
+                                Assert.Fail("Upload took too long. Blocks already expired.");
+                            }
+
+                            var dl = node.DownloadContent(cid);
+                            file.AssertIsEqual(dl);
+
+                            Measure("download-log-asserts", () =>
+                            {
+                                var downloadLog = node.DownloadLog(tailLines: 50000);
+
+                                var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
+                                downloadLog.DeleteFile();
+
+                                var sentLine = GetLineForCidTag(sentLines, cidTag);
+                                AssertSentLineContains(sentLine, sizeInBytes);
+                            });
+                            successfulDownloads++;
+                        });
+                    }
+                    catch
+                    {
+                        var testDuration = DateTime.UtcNow - startTime;
+                        Log("Test failed. Delaying shut-down by 30 seconds to collect metrics.");
+                        Log($"Test failed after {Time.FormatDuration(testDuration)} and {successfulUploads} successful uploads and {successfulDownloads} successful downloads");
+                        Thread.Sleep(TimeSpan.FromSeconds(30));
+                        throw;
+                    }
+                }
+
+                Thread.Sleep(TimeSpan.FromSeconds(5));
+            }
+        }
+
+        private void AssertSentLineContains(string sentLine, long sizeInBytes)
+        {
+            var tag = "bytes=";
+            var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length);
+            var bytes = Convert.ToInt64(token);
+            Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'");
+        }
+
+        private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes)
+        {
+            var tokens = storeLine.Split(" ");
+
+            var blocksToken = GetToken(tokens, "blocks=");
+            var sizeToken = GetToken(tokens, "size=");
+            if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine);
+            if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine);
+
+            var blocks = Convert.ToInt64(blocksToken);
+            var size = Convert.ToInt64(sizeToken?.Replace("'NByte", ""));
+
+            var lineLog = $" Line: '{storeLine}'";
+            Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog);
+            Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog);
+        }
+
+        private string GetLineForCidTag(string[] lines, string cidTag)
+        {
+            var result = lines.SingleOrDefault(l => l.Contains(cidTag));
+            if (result == null)
+            {
+                Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'");
+                throw new Exception();
+            }
+
+            return result;
+        }
+
+        private string? GetToken(string[] tokens, string tag)
+        {
+            var token = tokens.SingleOrDefault(t => t.StartsWith(tag));
+            if (token == null) return null;
+            return token.Substring(tag.Length);
+        }
+    }
+}

Tests/BasicTests/ExampleTests.cs

@@ -1,9 +1,5 @@
 using DistTestCore;
-using DistTestCore.Codex;
-using DistTestCore.Marketplace;
-using Newtonsoft.Json;
 using NUnit.Framework;
-using Utils;
 
 namespace Tests.BasicTests
 {

Tests/BasicTests/TwoClientTests.cs

@@ -8,7 +8,7 @@ namespace Tests.BasicTests
     public class TwoClientTests : DistTest
     {
         [Test]
-        public void TwoClientsOnePodTest()
+        public void TwoClientTest()
         {
             var group = SetupCodexNodes(2);
 
@@ -18,15 +18,6 @@ namespace Tests.BasicTests
             PerformTwoClientTest(primary, secondary);
         }
 
-        [Test]
-        public void TwoClientsTwoPodsTest()
-        {
-            var primary = SetupCodexNode();
-            var secondary = SetupCodexNode();
-
-            PerformTwoClientTest(primary, secondary);
-        }
-
         [Test]
         public void TwoClientsTwoLocationsTest()
         {

Tests/DownloadConnectivityTests/FulllyConnectedFetchTests.cs

@@ -0,0 +1,38 @@
+using DistTestCore;
+using NUnit.Framework;
+
+namespace Tests.DownloadConnectivityTests
+{
+    public class FulllyConnectedFetchTests : AutoBootstrapDistTest
+    {
+        [Test]
+        public void MetricsDoesNotInterfereWithFetch()
+        {
+            SetupCodexNodes(2, s => s.EnableMetrics());
+
+            AssertAllNodesConnected();
+        }
+
+        [Test]
+        public void MarketplaceDoesNotInterfereWithFetch()
+        {
+            SetupCodexNodes(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens()));
+
+            AssertAllNodesConnected();
+        }
+
+        [Test]
+        [Combinatorial]
+        public void FullyConnectedFetchTest([Values(1, 3, 5)] int numberOfNodes)
+        {
+            SetupCodexNodes(numberOfNodes);
+
+            AssertAllNodesConnected();
+        }
+
+        private void AssertAllNodesConnected()
+        {
+            CreatePeerFetchTestHelpers().AssertFullFetchInterconnectivity(GetAllOnlineCodexNodes());
+        }
+    }
+}

Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -6,15 +6,36 @@ namespace Tests.DownloadConnectivityTests
     [TestFixture]
     public class FullyConnectedDownloadTests : AutoBootstrapDistTest
     {
+        [Test]
+        public void MetricsDoesNotInterfereWithPeerDownload()
+        {
+            SetupCodexNodes(2, s => s.EnableMetrics());
+
+            AssertAllNodesConnected();
+        }
+
+        [Test]
+        public void MarketplaceDoesNotInterfereWithPeerDownload()
+        {
+            SetupCodexNodes(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens()));
+
+            AssertAllNodesConnected();
+        }
+
         [Test]
         [Combinatorial]
         public void FullyConnectedDownloadTest(
             [Values(1, 3, 5)] int numberOfNodes,
             [Values(1, 10)] int sizeMBs)
         {
-            for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
+            SetupCodexNodes(numberOfNodes);
 
-            PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
+            AssertAllNodesConnected(sizeMBs);
+        }
+
+        private void AssertAllNodesConnected(int sizeMBs = 10)
+        {
+            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
         }
     }
 }

Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs

@@ -1,5 +1,4 @@
 using DistTestCore;
-using DistTestCore.Helpers;
 using NUnit.Framework;
 
 namespace Tests.PeerDiscoveryTests
@@ -47,7 +46,7 @@ namespace Tests.PeerDiscoveryTests
 
         private void AssertAllNodesConnected()
         {
-            PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
+            CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
         }
     }
 }

Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -1,5 +1,4 @@
 using DistTestCore;
-using DistTestCore.Helpers;
 using NUnit.Framework;
 
 namespace Tests.PeerDiscoveryTests
@@ -17,23 +16,36 @@ namespace Tests.PeerDiscoveryTests
             Assert.That(result.IsPeerFound, Is.False);
         }
 
+        [Test]
+        public void MetricsDoesNotInterfereWithPeerDiscovery()
+        {
+            SetupCodexNodes(2, s => s.EnableMetrics());
+
+            AssertAllNodesConnected();
+        }
+
+        [Test]
+        public void MarketplaceDoesNotInterfereWithPeerDiscovery()
+        {
+            SetupCodexNodes(2, s => s.EnableMarketplace(1000.TestTokens()));
+
+            AssertAllNodesConnected();
+        }
+
         [TestCase(2)]
         [TestCase(3)]
         [TestCase(10)]
         [TestCase(20)]
-        public void VariableNodesInPods(int number)
+        public void VariableNodes(int number)
         {
-            for (var i = 0; i < number; i++)
-            {
-                SetupCodexNode();
-            }
+            SetupCodexNodes(number);
 
             AssertAllNodesConnected();
         }
 
         private void AssertAllNodesConnected()
         {
-            PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
+            CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
         }
     }
 }

Tests/Tests.csproj

@@ -13,6 +13,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\ContinuousTests\ContinuousTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
   </ItemGroup>
 

docker/README.md

@@ -1,138 +0,0 @@
-# Run tests with Docker in Kubernetes
-
- We may [run tests on local](../LOCALSETUP.MD) or remote Kubernetes cluster. Local cluster flow uses direct access to nodes ports and this is why we introduced a different way to check services ports, for more information please see [Tests run modes](../../../issues/20). Configuration option `RUNNERLOCATION` is responsible for that.
-
-
- For local run it is easier to install .Net and run tests on Docker Desktop Kubernetes cluster. In case of remote run we do not expose services via Ingress Controller and we can't access cluster nodes, this is why we should run tests only inside the Kubernetes.
-
- We can run tests on remote cluster in the following ways
-
-#### Run pod inside the cluster using generic .Net image
-
-<details>
-<summary>steps</summary>
-
-1. Create dist-tests-runner.yaml
-   ```yaml
-      --
-      apiVersion: v1
-      kind: Pod
-      metadata:
-      name: dist-tests-runner
-      namespace: default
-      spec:
-      containers:
-      - name: dotnet
-         image: mcr.microsoft.com/dotnet/sdk:7.0
-         command: ["sleep", "infinity"]
-   ```
-2. Deploy pod in the cluster
-   ```shell
-   kubectl apply -f dist-tests-runner.yaml
-   ```
-
-3. Copy kubeconfig to the pod
-   ```shell
-   kubectl cp kubeconfig.yaml dist-tests-runner:/opt
-   ```
-
-4. Exec into the pod via kubectl or [OpenLens](https://github.com/MuhammedKalkan/OpenLens)
-   ```shell
-   kubectl exec -it dist-tests-runner -- bash
-   ```
-
-5. Clone repository inside the pod
-   ```shell
-   git clone https://github.com/codex-storage/cs-codex-dist-tests.git
-   ```
-
-6. Update kubeconfig option in config file
-   ```shell
-   cd cs-codex-dist-tests
-   vi DistTestCore/Configuration.cs
-   ```
-   ```dotnet
-   GetNullableEnvVarOrDefault("KUBECONFIG", "/opt/kubeconfig.yaml")
-   ```
-
-7. Run tests
-   ```shell
-   dotnet test Tests
-   ```
-
-8. Check the results and analyze the logs
-</details>
-
-#### Run pod inside the cluster using [prepared Docker images](https://hub.docker.com/r/codexstorage/cs-codex-dist-tests/tags)
-
- Before the run we should create some objects inside the cluster
- 1. Namespace where we will run the image
- 2. [Service Account to run tests inside the cluster](https://github.com/codex-storage/cs-codex-dist-tests/issues/21)
- 3. Secret with kubeconfig for created SA
- 4. Configmap with custom app config if required
-
- For more information please see [Manual run inside Kubernetes via Job](../../../issues/7)
-
- Then we need to create a manifest to run the pod
- <details>
- <summary>runner.yaml</summary>
-
- ```yaml
- ---
- apiVersion: v1
- kind: Pod
- metadata:
-   name: dist-tests-runner
-   namespace: cs-codex-dist-tests
-   labels:
-     name: cs-codex-dist-tests
- spec:
-   containers:
-   - name: cs-codex-dist-tests
-     image: codexstorage/cs-codex-dist-tests:sha-671ee4e
-     env:
-     - name: RUNNERLOCATION
-       value: InternalToCluster
-     - name: KUBECONFIG
-       value: /opt/kubeconfig.yaml
-     - name: CONFIG
-       value: "/opt/Configuration.cs"
-     - name: CONFIG_SHOW
-       value: "true"
-     volumeMounts:
-     - name: kubeconfig
-       mountPath: /opt/kubeconfig.yaml
-       subPath: kubeconfig.yaml
-     - name: config
-       mountPath: /opt/Configuration.cs
-       subPath: Configuration.cs
-     - name: logs
-       mountPath: /var/log/cs-codex-dist-tests
-     # command:
-     # - "dotnet"
-     # - "test"
-     # - "Tests"
-   restartPolicy: Never
-   volumes:
-     - name: kubeconfig
-       secret:
-         secretName: cs-codex-dist-tests-app-kubeconfig
-     - name: config
-       configMap:
-         name: cs-codex-dist-tests
-     - name: logs
-       hostPath:
-         path: /var/log/cs-codex-dist-tests
- ```
- For more information about pod variables please see [job.yaml](job.yaml).
- </details>
-
- And then apply it
- ```shell
- kubectl apply -f runner.yaml
- ```
-
- After the pod run, custom [entrypoint](docker-entrypoint.sh) will do the following
- 1. Clone repository
- 2. Switch to the specific branch - `master` by default
- 3. Run all tests - `dotnet test`

docker/job.yaml

@@ -14,7 +14,8 @@ spec:
     spec:
       containers:
       - name: ${NAMEPREFIX}-runner
-        image: codexstorage/cs-codex-dist-tests:sha-300b91e
+        image: codexstorage/cs-codex-dist-tests:latest
+        imagePullPolicy: Always
         env:
         - name: RUNNERLOCATION
           value: InternalToCluster

docs/AUTOMATION.md

@@ -0,0 +1,253 @@
+# Distributed Tests automation
+
+ 1. [Description](#description)
+ 2. [Architecture](#architecture)
+ 3. [Run on remote cluster](#run-on-remote-cluster)
+    - [Kubernetes cluster pre-configuration](#kubernetes-cluster-pre-configuration)
+    - [Run tests manually](#run-tests-manually)
+    - [Run tests automatically](#run-tests-automatically)
+    - [Tests logs](#tests-logs)
+
+
+## Description
+
+ The goal of [Distributed System Tests for Nim-Codex](../../) is to test how [Codex](https://codex.storage) works in different topologies in the distributed network and to be able to detect regressions during development.
+
+ We can [run Tests locally](LOCALSETUP.md) and it works well, but in order to scale that we may need to run Tests in an automatic way using remote Kubernetes cluster.
+
+ Initially, we are considering to run dist-tests on [nim-codex](https://github.com/codex-storage/nim-codex) master branch merge, to be able to determine regressions. And we also working on [Continuous Tests](/ContinuousTests) which are called to detect issues on continuous Codex runs.
+
+
+## Architecture
+
+<img src="Architecture.png" alt="Architecture" width="800"/>
+
+```
+                                        Logs     --> Kibana
+                                      /                |
+GitHub --> CI --> Kubernetes -->  Job    Prometheus  Elaticsearch
+      \                          /    \  /         \   |
+        ------------------------        Metrics  --> Grafana
+```
+
+
+### Components
+
+| Component                               | Description                             |
+| --------------------------------------- | --------------------------------------- |
+| [cs-codex-dist-tests](/)                | Distributed System Tests                |
+| [Kubernetes](https://kubernetes.io)     | Environment where we run Tests          |
+| [Vector](https://vector.dev)            | Ship logs to the Elasticsearch          |
+| [Elasticsearch](https://www.elastic.co) | Store and index logs                    |
+| [Kibana](https://www.elastic.co)        | Discover the logs in Elasticsearch      |
+| [Grafana](https://grafana.com)          | Visualize tests run results and metrics |
+| [Prometheus](https://prometheus.io)     | Collect and store metrics               |
+
+> Note: These components are not mentioned on the diagram and provided to understand what do we have under the hood
+
+
+## Run on remote cluster
+
+ In case of local run we use [Docker Desktop Kubernetes cluster](https://docs.docker.com/desktop/kubernetes/) and during the services checks, app connect directly to the cluster worker nodes and perform ports check. And in case of remote cluster, it would be required to configure services ports exposing using Ingress Controller or run tests directly inside the cluster.
+
+ Now, we have a configuration key `RUNNERLOCATION` which change the logic of the services ports check and when we run tests inside remote cluster we should set it to the `InternalToCluster`.
+
+ As for now, it was decided to run tests inside the remote Kubernetes cluster using CI because
+ - Stable connection from app to the Kubernetes API
+ - Independence from the CI limitations for long runs
+ - Easy, fast, configurable and detachable run
+
+ Because tests are run on remote cluster we need a way to see their execution status and to analyze the logs as well. For that we use Elasticsearch, Kibana and Grafana with logs shipped by Vector. Please see [Tests logs](#tests-logs) for more information.
+
+ Now we can [Run tests manually](#run-tests-manually) and [Run tests automatically](#run-tests-automatically) on [remote Kubernetes cluster which requires to be pre-configured](#kubernetes-cluster-pre-configuration).
+
+
+### Kubernetes cluster pre-configuration
+
+ Before running the tests on remote Kubernetes cluster we performed [manual pre-configuration](../../../issues/7) and it was require to
+ 1. [Create a namespace](../../../issues/7)
+ 2. [Create kubeconfig for App](../../../issues/21)
+ 3. [Create a secret with created kubeconfig](../../../issues/21)
+ 4. [Create kubeconfig for GitHub Actions](../../../issues/19)
+
+
+### Run tests manually
+
+ **To run tests manually we have the following requirements**
+ 1. Get kubeconfig - to access the cluster
+ 2. Install [kubectl](https://kubernetes.io/docs/tasks/tools/) - to create resources in the cluster
+ 3. Install [OpenLens](https://github.com/MuhammedKalkan/OpenLens) - to browse cluster resources
+
+ **And to run the tests we should perform the following steps**
+ 1. Create a Pod in the cluster, in the `default` namespace and consider to use your own value for `metadata.name`
+    <details>
+    <summary>pod.yaml</summary>
+
+    ```yaml
+    ---
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: dist-tests-runner
+      namespace: default
+      labels:
+        app: dist-tests-runner
+        launched: manually
+    spec:
+      containers:
+      - name: dotnet
+        image: mcr.microsoft.com/dotnet/sdk:7.0
+        env:
+        - name: RUNNERLOCATION
+          value: InternalToCluster
+        - name: KUBECONFIG
+          value: /opt/kubeconfig.yaml
+        command: ["sleep", "infinity"]
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /opt/kubeconfig.yaml
+          subPath: kubeconfig.yaml
+        - name: logs
+          mountPath: /var/log/cs-codex-dist-tests
+      restartPolicy: Never
+      volumes:
+        - name: kubeconfig
+          secret:
+            secretName: cs-codex-dist-tests-app-kubeconfig
+        - name: logs
+          hostPath:
+            path: /var/log/cs-codex-dist-tests
+    ```
+    </details>
+
+    ```shell
+    kubectl apply -f pod.yaml
+    ```
+
+ 2. Exec into the Pod using the name you set in the previous step
+    ```shell
+    # kubectl
+    kubectl exec -it dist-tests-runner -- bash
+
+    # OpenLens
+    OpenLens --> Pods --> dist-tests-runner --> "Press on it" --> Pod Shell
+    ```
+
+ 3. Clone dist-tests repository
+    ```shell
+    folder="/opt/dist-tests"
+
+    git clone https://github.com/codex-storage/cs-codex-dist-tests.git $folder
+    cd $folder
+    ```
+
+ 4. Define variables - optional
+    ```shell
+    # RUNNERLOCATION                              # defined at Pod creation
+    # KUBECONFIG                                  # defined at Pod creation
+    export LOGPATH="/var/log/cs-codex-dist-tests" # Logs from that location will be send in Elasticsearch
+    export RUNID=$(date +%Y%m%d-%H%M%S)           # Run ID to show in Kibana/Grafana
+    export TESTID=$(git rev-parse --short HEAD)   # Test ID to show in Kibana/Grafana
+    ```
+
+ 5. Run tests
+    ```shell
+    # All tests
+    dotnet test
+
+    # Short tests
+    dotnet test Tests
+
+    # Long tests
+    dotnet test LongTests
+
+    # Specific test
+    dotnet test --filter=CodexLogExample
+    ```
+
+ 6. We can see in OpenLens Pods started by dist-tests app
+
+ 7. If we set `LOGPATH` to a location specified above, we should be able to see tests execution status in Kibana/Grafana. For more information, please see [Tests logs](#tests-logs).
+
+
+### Run tests automatically
+
+ Now we use GitHub Actions to trigger dist-tests run manually and considering to run them on [nim-codex](https://github.com/codex-storage/nim-codex) master branch merge.
+
+ **It works in the following way**
+ 1. Github Actions secrets contains [kubeconfig to interact with the Kubernetes cluster](../../../issues/19).
+ 2. GitHub Actions [workflow](../../../actions/workflows/dist-tests.yaml) uses kubectl to create Kubernetes Job based on the [job.yaml](/docker/job.yaml) manifest file. It also accept optional inputs at run
+    - `source`     - Dist-tests source repository
+    - `branch`     - Repository branch
+    - `namespace`  - Namespace where Dist-test runner will de created
+    - `nameprefix` - Dist-test runner name prefix
+
+ 3. Kubernetes Job will run the Pod with a custom [Docker image](/docker/Dockerfile) - [codexstorage/cs-codex-dist-tests](https://hub.docker.com/r/codexstorage/cs-codex-dist-tests/tags).  
+  Image [entrypoint](/docker/docker-entrypoint.sh) is customizable and we can pass the following variables
+    - `SOURCE` - Dist-tests source repository, useful when we work with forks - `default="current repository"`
+    - `BRANCH` - Repository branch, useful when we would like to run tests from the custom branch - `default="master"`
+    - `FOLDER` - Where to clone repository and it is done just to organize the things - `default="/opt/dist-tests"`
+
+    > **Note:** Variables `SOURCE` and `BRANCH` passed by GitHub Actions to the Kubernetes Job and then to the Pod.
+
+ 4. Job manifest is setting all required variables which are part of the Pod run
+    ```shell
+    RUNNERLOCATION=InternalToCluster
+    KUBECONFIG=/opt/kubeconfig.yaml
+    LOGPATH=/var/log/cs-codex-dist-tests
+    NAMESPACE=default
+    SOURCE=current repository
+    BRANCH=master
+    RUNID=datetime
+    TESTID=short sha
+    ```
+
+ 5. Dist-tests runner will use [kubeconfig to interact with the Kubernetes and run the tests](../../../issues/21), which is set by `KUBECONFIG` variable.
+
+ 6. Runner will execute all tests and will write the logs to the `/var/log/cs-codex-dist-tests` folder.
+
+ 7. Kubernetes Job status will be changed from `Running` to the `Completed` or `Failed`.
+
+ 8. Vector will ship the logs to the Elasticsearch.
+
+ 9. We can check execution status in Kibana and Grafana. For more information, please see [Tests logs](#tests-logs).
+
+
+### Tests logs
+
+ > **Note:** This part is not finished yet and it is under development
+
+ We use Elasticsearch to store and discover the logs of the tests execution and Codex Pods which are run during the tests.
+
+ We can use [Kibana](#kibana) to discover all the logs and [Grafana](#grafana) to see tests execution status.
+
+
+#### Endpoints
+
+ | App     | URL                                                                          | Authentication |
+ | ------- | ---------------------------------------------------------------------------- | -------------- |
+ | Kibana  | [kibana.dist-tests.codex.storage](https://kibana.dist-tests.codex.storage)   | GitHub account |
+ | Grafana | [grafana.dist-tests.codex.storage](https://grafana.dist-tests.codex.storage) | GitHub account |
+
+
+#### Kibana
+
+ As for now, we have the following indices in Kibana
+ | # | Index Pattern              | Description                         |
+ | - | -------------------------- | ----------------------------------- |
+ | 1 | `dist-tests-status*`       | Dist-tests execution status logs    |
+ | 2 | `dist-tests-logs*`         | Dist-tests logs collected by runner |
+ | 3 | `dist-tests-pods*`         | Dist-tests Pods logs                |
+ | 4 | `continuous-tests-status*` | Dist-tests execution status logs    |
+ | 5 | `continuous-tests-logs*`   | Dist-tests logs collected by runner |
+ | 6 | `continuous-tests-pods*`   | Dist-tests Pods logs                |
+ | 7 | `kubernetes*`              | All Kubernetes Pods logs            |
+
+
+#### Grafana
+
+ As for now, we have the following Dashboards in Grafana
+ | # | Dashboard                  |
+ | - | -------------------------- |
+ | 1 | Distributed Tests - Status |
+ | 2 | Continuous Tests - Status  |