Removes log asserts, they are unreliable in the cluster.

# Conflicts:
#	DistTestCore/Codex/CodexContainerRecipe.cs

.github/workflows/docker-reusable.yml

@@ -0,0 +1,177 @@
+name: Docker - Reusable
+
+
+on:
+  workflow_call:
+    inputs:
+      docker_file:
+        default: docker/Dockerfile
+        description: Dockerfile
+        required: false
+        type: string
+      docker_repo:
+        default: codexstorage/cs-codex-dist-tests
+        description: DockerHub repository
+        required: false
+        type: string
+      tag_latest:
+        default: true
+        description: Set latest tag for Docker images
+        required: false
+        type: boolean
+      tag_sha:
+        default: true
+        description: Set Git short commit as Docker tag
+        required: false
+        type: boolean
+      tag_suffix:
+        default: ''
+        description: Suffix for Docker images tag
+        required: false
+        type: string
+
+
+env:
+  DOCKER_FILE: ${{ inputs.docker_file }}
+  DOCKER_REPO: ${{ inputs.docker_repo }}
+  TAG_LATEST: ${{ inputs.tag_latest }}
+  TAG_SHA: ${{ inputs.tag_sha }}
+  TAG_SUFFIX: ${{ inputs.tag_suffix }}
+
+
+jobs:
+  # Build platform specific image
+  build:
+    strategy:
+      fail-fast: true
+      matrix:
+        target:
+          - os: linux
+            arch: amd64
+          - os: linux
+            arch: arm64
+        include:
+          - target:
+              os: linux
+              arch: amd64
+            builder: ubuntu-22.04
+          - target:
+              os: linux
+              arch: arm64
+            builder: buildjet-4vcpu-ubuntu-2204-arm
+
+    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
+    runs-on: ${{ matrix.builder }}
+    env:
+      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_REPO }}
+
+      - name: Docker - Set up Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Build and Push by digest
+        id: build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ${{ env.DOCKER_FILE }}
+          platforms: ${{ env.PLATFORM }}
+          push: true
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Docker - Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Docker - Upload digest
+        uses: actions/upload-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+
+  # Publish multi-platform image
+  publish:
+    name: Publish multi-platform image
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Docker - Variables
+        run: |
+          # Adjust custom suffix when set and
+          if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
+            echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
+          fi
+          # Disable SHA tags on tagged release
+          if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
+            echo "TAG_SHA=false" >>$GITHUB_ENV
+          fi
+          # Handle latest and latest-custom using raw
+          if [[ ${{ env.TAG_SHA }} == "false" ]]; then
+            echo "TAG_LATEST=false" >>$GITHUB_ENV
+            echo "TAG_RAW=true" >>$GITHUB_ENV
+            if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
+              echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
+            else
+              echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
+            fi
+          else
+            echo "TAG_RAW=false" >>$GITHUB_ENV
+          fi
+
+      - name: Docker - Download digests
+        uses: actions/download-artifact@v3
+        with:
+          name: digests
+          path: /tmp/digests
+
+      - name: Docker - Set up Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_REPO }}
+          flavor: |
+            latest=${{ env.TAG_LATEST }}
+            suffix=${{ env.TAG_SUFFIX }},onlatest=true
+          tags: |
+            type=semver,pattern={{version}}
+            type=raw,enable=${{ env.TAG_RAW }},value=latest
+            type=sha,enable=${{ env.TAG_SHA }}
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
+
+      - name: Docker - Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}

.github/workflows/docker.yml

@@ -11,113 +11,12 @@ on:
       - docker/Dockerfile
       - docker/docker-entrypoint.sh
       - .github/workflows/docker.yml
+      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
 
-env:
-  DOCKER_FILE: docker/Dockerfile
-  DOCKER_REPO: codexstorage/cs-codex-dist-tests
-
-
 jobs:
-  # Build platform specific image
-  build:
-    strategy:
-      fail-fast: true
-      matrix:
-        target:
-          - os: linux
-            arch: amd64
-          - os: linux
-            arch: arm64
-        include:
-          - target:
-              os: linux
-              arch: amd64
-            builder: ubuntu-22.04
-          - target:
-              os: linux
-              arch: arm64
-            builder: buildjet-4vcpu-ubuntu-2204-arm
-
-    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
-    runs-on: ${{ matrix.builder }}
-    outputs:
-      tags-linux-amd64: ${{ steps.tags.outputs.tags-linux-amd64 }}
-      tags-linux-arm64: ${{ steps.tags.outputs.tags-linux-arm64 }}
-    env:
-      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
-      SUFFIX: ${{ format('{0}-{1}', 'linux', matrix.target.arch) }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.DOCKER_REPO }}
-          flavor: |
-            latest=false
-          tags: |
-            type=semver,pattern={{version}},suffix=-${{ env.SUFFIX }}
-            type=sha,suffix=-${{ env.SUFFIX }},enable=${{ !startsWith(github.ref, 'refs/tags/') }}
-
-      - name: Docker - Set tags output
-        id: tags
-        run: |
-          if [[ '${{ matrix.target.os }}' == 'linux' && '${{ matrix.target.arch }}' == 'amd64' ]]; then
-            echo "tags-linux-amd64=${{ steps.meta.outputs.tags }}" >> "$GITHUB_OUTPUT"
-          elif [[ '${{ matrix.target.os }}' == 'linux' && '${{ matrix.target.arch }}' == 'arm64' ]]; then
-            echo "tags-linux-arm64=${{ steps.meta.outputs.tags }}" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Build and Push
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ${{ env.DOCKER_FILE }}
-          platforms: ${{ env.PLATFORM }}
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-
-  # Publish single image
-  publish:
-    name: Push single image
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.DOCKER_REPO }}
-          tags: |
-            type=semver,pattern={{version}}
-            type=sha,enable=${{ !startsWith(github.ref, 'refs/tags/') }}
-
-      - name: Docker - Set tags
-        run: |
-          # Transform multi-line tags in to the comma-seperated
-          TAGS=$(echo "${{ steps.meta.outputs.tags }}" | tr '\n' ',' | awk '{gsub(/,$/,"");}1')
-          echo "TAGS=${TAGS}" >>$GITHUB_ENV
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Create and push manifest images
-        uses: Noelware/docker-manifest-action@master
-        with:
-          inputs: ${{ env.TAGS }}
-          images: ${{ needs.build.outputs.tags-linux-amd64 }},${{ needs.build.outputs.tags-linux-arm64 }}
-          push: true
+  build-and-push:
+    name: Build and Push
+    uses: ./.github/workflows/docker-reusable.yml
+    secrets: inherit

CodexNetDeployer/CodexNodeStarter.cs

@@ -8,25 +8,23 @@ namespace CodexNetDeployer
     public class CodexNodeStarter
     {
         private readonly Configuration config;
-        private readonly WorkflowCreator workflowCreator;
         private readonly TestLifecycle lifecycle;
         private readonly GethStartResult gethResult;
         private string bootstrapSpr = "";
         private int validatorsLeft;
 
-        public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
+        public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
         {
             this.config = config;
-            this.workflowCreator = workflowCreator;
             this.lifecycle = lifecycle;
             this.gethResult = gethResult;
             validatorsLeft = numberOfValidators;
         }
 
-        public RunningContainer? Start(int i)
+        public CodexNodeStartResult? Start(int i)
         {
             Console.Write($" - {i} = ");
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var workflowStartup = new StartupConfig();
             workflowStartup.Add(gethResult);
             workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));
@@ -62,7 +60,7 @@ namespace CodexNetDeployer
 
                         if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
                         validatorsLeft--;
-                        return container;
+                        return new CodexNodeStartResult(workflow, container, codexAccess);
                     }
                 }
             }
@@ -86,14 +84,36 @@ namespace CodexNetDeployer
             var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
             marketplaceConfig.AccountIndexOverride = i;
             codexStart.MarketplaceConfig = marketplaceConfig;
-            codexStart.MetricsEnabled = config.RecordMetrics;
+            codexStart.MetricsMode = config.Metrics;
 
             if (config.BlockTTL != Configuration.SecondsIn1Day)
             {
                 codexStart.BlockTTL = config.BlockTTL;
             }
+            if (config.BlockMI != Configuration.TenMinutes)
+            {
+                codexStart.BlockMaintenanceInterval = TimeSpan.FromSeconds(config.BlockMI);
+            }
+            if (config.BlockMN != 1000)
+            {
+                codexStart.BlockMaintenanceNumber = config.BlockMN;
+            }
 
             return codexStart;
         }
     }
+
+    public class CodexNodeStartResult
+    {
+        public CodexNodeStartResult(StartupWorkflow workflow, RunningContainer container, CodexAccess access)
+        {
+            Workflow = workflow;
+            Container = container;
+            Access = access;
+        }
+
+        public StartupWorkflow Workflow { get; }
+        public RunningContainer Container { get; }
+        public CodexAccess Access { get; }
+    }
 }

CodexNetDeployer/Configuration.cs

@@ -1,12 +1,13 @@
 using ArgsUniform;
-using DistTestCore;
 using DistTestCore.Codex;
+using DistTestCore.Metrics;
 
 namespace CodexNetDeployer
 {
     public class Configuration
     {
         public const int SecondsIn1Day = 24 * 60 * 60;
+        public const int TenMinutes = 10 * 60;
 
         [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
         public string KubeConfigFile { get; set; } = "null";
@@ -44,11 +45,22 @@ namespace CodexNetDeployer
         [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
         public int BlockTTL { get; set; } = SecondsIn1Day;
 
-        [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
-        public bool RecordMetrics { get; set; } = false;
-       
-        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
+        [Uniform("block-mi", "bmi", "BLOCKMI", false, "Block maintenance interval in seconds. Default is 10 minutes.")]
+        public int BlockMI { get; set; } = TenMinutes;
 
+        [Uniform("block-mn", "bmn", "BLOCKMN", false, "Number of blocks maintained per interval. Default is 1000 blocks.")]
+        public int BlockMN { get; set; } = 1000;
+
+        [Uniform("metrics", "m", "METRICS", false, "[None*, Record, Dashboard]. Determines if metrics will be recorded and if a dashboard service will be created.")]
+        public MetricsMode Metrics { get; set; } = MetricsMode.None;
+
+        [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
+            "set this option to override the label value.")]
+        public string TestsTypePodLabel { get; set; } = "continuous-tests";
+
+        [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment.")]
+        public bool CheckPeerConnection { get; set; } = false;
+       
         public List<string> Validate()
         {
             var errors = new List<string>();

CodexNetDeployer/Deployer.cs

@@ -8,29 +8,29 @@ namespace CodexNetDeployer
     public class Deployer
     {
         private readonly Configuration config;
-        private readonly NullLog log;
         private readonly DefaultTimeSet timeset;
+        private readonly PeerConnectivityChecker peerConnectivityChecker;
 
         public Deployer(Configuration config)
         {
             this.config = config;
-            log = new NullLog();
             timeset = new DefaultTimeSet();
+            peerConnectivityChecker = new PeerConnectivityChecker();
         }
 
         public CodexDeployment Deploy()
         {
             Log("Initializing...");
-            var (workflowCreator, lifecycle) = CreateFacilities();
+            var lifecycle = CreateTestLifecycle();
 
             Log("Preparing configuration...");
             // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
             var setup = new CodexSetup(999, config.CodexLogLevel);
             setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
-            setup.MetricsEnabled = config.RecordMetrics;
+            setup.MetricsMode = config.Metrics;
 
             Log("Creating Geth instance and deploying contracts...");
-            var gethStarter = new GethStarter(lifecycle, workflowCreator);
+            var gethStarter = new GethStarter(lifecycle);
             var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
 
             Log("Geth started. Codex contracts deployed.");
@@ -44,20 +44,23 @@ namespace CodexNetDeployer
             Log("Starting Codex nodes...");
 
             // Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
-            var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, gethResults, config.NumberOfValidators!.Value);
-            var codexContainers = new List<RunningContainer>();
+            var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
+            var startResults = new List<CodexNodeStartResult>();
             for (var i = 0; i < config.NumberOfCodexNodes; i++)
             {
-                var container = codexStarter.Start(i);
-                if (container != null) codexContainers.Add(container);
+                var result = codexStarter.Start(i);
+                if (result != null) startResults.Add(result);
             }
 
-            var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers);
+            var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container));
 
-            return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
+            CheckPeerConnectivity(startResults);
+            CheckContainerRestarts(startResults);
+
+            return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
         }
 
-        private (WorkflowCreator, TestLifecycle) CreateFacilities()
+        private TestLifecycle CreateTestLifecycle()
         {
             var kubeConfig = GetKubeConfig(config.KubeConfigFile);
 
@@ -68,28 +71,25 @@ namespace CodexNetDeployer
                 logDebug: false,
                 dataFilesPath: "notUsed",
                 codexLogLevel: config.CodexLogLevel,
-                runnerLocation: config.RunnerLocation
+                k8sNamespacePrefix: config.KubeNamespace
             );
 
-            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: config.KubeNamespace,
-                kubeConfigFile: kubeConfig,
-                operationTimeout: timeset.K8sOperationTimeout(),
-                retryDelay: timeset.WaitForK8sServiceDelay());
-
-            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
-            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
-
-            return (workflowCreator, lifecycle);
+            return new TestLifecycle(new NullLog(), lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
         }
 
-        private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
+        private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable<RunningContainer> codexContainers)
         {
-            if (!setup.MetricsEnabled) return null;
+            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
 
             Log("Starting metrics service...");
-            var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray());
-            return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
+            var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
+            var prometheusContainer = lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
+
+            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null);
+
+            Log("Starting dashboard service...");
+            var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer, setup);
+            return (prometheusContainer, grafanaStartInfo);
         }
 
         private string? GetKubeConfig(string kubeConfigFile)
@@ -98,6 +98,35 @@ namespace CodexNetDeployer
             return kubeConfigFile;
         }
 
+        private void CheckPeerConnectivity(List<CodexNodeStartResult> codexContainers)
+        {
+            if (!config.CheckPeerConnection) return;
+
+            Log("Starting peer-connectivity check for deployed nodes...");
+            peerConnectivityChecker.CheckConnectivity(codexContainers);
+            Log("Check passed.");
+        }
+
+        private void CheckContainerRestarts(List<CodexNodeStartResult> startResults)
+        {
+            var crashes = new List<RunningContainer>();
+            foreach (var startResult in startResults)
+            {
+                var watcher = startResult.Workflow.CreateCrashWatcher(startResult.Container);
+                if (watcher.HasContainerCrashed()) crashes.Add(startResult.Container);
+            }
+
+            if (!crashes.Any())
+            {
+                Log("Container restart check passed.");
+            }
+            else
+            {
+                Log($"Deployment failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}");
+                throw new Exception("Deployment failed: One or more containers crashed.");
+            }
+        }
+
         private DeploymentMetadata CreateMetadata()
         {
             return new DeploymentMetadata(
@@ -109,7 +138,10 @@ namespace CodexNetDeployer
                 initialTestTokens: config.InitialTestTokens,
                 minPrice: config.MinPrice,
                 maxCollateral: config.MaxCollateral,
-                maxDuration: config.MaxDuration);
+                maxDuration: config.MaxDuration,
+                blockTTL: config.BlockTTL,
+                blockMI: config.BlockMI,
+                blockMN: config.BlockMN);
         }
 
         private void Log(string msg)

CodexNetDeployer/PeerConnectivityChecker.cs

@@ -0,0 +1,34 @@
+using DistTestCore.Helpers;
+using Logging;
+
+namespace CodexNetDeployer
+{
+    public class PeerConnectivityChecker
+    {
+        public void CheckConnectivity(List<CodexNodeStartResult> startResults)
+        {
+            var log = new ConsoleLog();
+            var checker = new PeerConnectionTestHelpers(log);
+            var access = startResults.Select(r => r.Access);
+
+            checker.AssertFullyConnected(access);
+        }
+    }
+
+    public class ConsoleLog : BaseLog
+    {
+        public ConsoleLog() : base(false)
+        {
+        }
+
+        protected override string GetFullName()
+        {
+            return "CONSOLE";
+        }
+
+        public override void Log(string message)
+        {
+            Console.WriteLine(message);
+        }
+    }
+}

CodexNetDeployer/Program.cs

@@ -1,6 +1,5 @@
 using ArgsUniform;
 using CodexNetDeployer;
-using DistTestCore;
 using DistTestCore.Codex;
 using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
@@ -17,11 +16,6 @@ public class Program
         var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
         var config = uniformArgs.Parse(true);
         
-        if (args.Any(a => a == "--external"))
-        {
-            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
-        }
-
         var errors = config.Validate();
         if (errors.Any())
         {
@@ -36,7 +30,8 @@ public class Program
             $"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl +
             $"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl +
             $"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl +
-            $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl);
+            $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl +
+            $"\tGrafana image: '{new GrafanaContainerRecipe().Image}'" + nl);
 
         if (!args.Any(a => a == "-y"))
         {
@@ -61,8 +56,5 @@ public class Program
         Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
             "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
             "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
-
-        Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
-            "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
     }
 }

CodexNetDeployer/deploy-continuous-testnet.sh

@@ -9,5 +9,8 @@ dotnet run \
     --min-price=1024 \
     --max-collateral=1024 \
     --max-duration=3600000 \
-    --block-ttl=120 \
-	-y
+    --block-ttl=180 \
+    --block-mi=120 \
+    --block-mn=10000 \
+    --metrics=Dashboard \
+    --check-connect=1

CodexNetDownloader/Configuration.cs

@@ -1,5 +1,4 @@
 using ArgsUniform;
-using DistTestCore;
 using DistTestCore.Codex;
 
 namespace CodexNetDownloader
@@ -16,7 +15,5 @@ namespace CodexNetDownloader
         public string KubeConfigFile { get; set; } = "null";
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
-
-        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
     }
 }

CodexNetDownloader/Program.cs

@@ -15,17 +15,12 @@ public class Program
         var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(PrintHelp, args);
         var config = uniformArgs.Parse(true);
 
-        if (args.Any(a => a == "--external"))
-        {
-            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
-        }
-
         config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
 
         if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
 
         var k8sFactory = new K8sFactory();
-        var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+        var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
 
         foreach (var container in config.CodexDeployment.CodexContainers)
         {

ContinuousTests/CodexAccessFactory.cs

@@ -12,7 +12,7 @@ namespace ContinuousTests
             return containers.Select(container =>
             {
                 var address = container.ClusterExternalAddress;
-                if (config.RunnerLocation == TestRunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
+                if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
                 return new CodexAccess(log, container, timeSet, address);
             }).ToArray();
         }

ContinuousTests/Configuration.cs

@@ -30,7 +30,7 @@ namespace ContinuousTests
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
 
-        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
+        public RunnerLocation RunnerLocation { get; set; }
     }
 
     public class ConfigLoader
@@ -42,10 +42,7 @@ namespace ContinuousTests
             var result = uniformArgs.Parse(true);
             
             result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
-            if (args.Any(a => a == "--external"))
-            {
-                result.RunnerLocation = TestRunnerLocation.ExternalToCluster;
-            }
+            result.RunnerLocation = RunnerLocationUtils.DetermineRunnerLocation(result.CodexDeployment.CodexContainers.First());
 
             return result;
         }
@@ -63,7 +60,6 @@ namespace ContinuousTests
             Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
                 "The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
 
-
             Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
                 "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
         }

ContinuousTests/ContinuousLogDownloader.cs

@@ -1,99 +0,0 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using KubernetesWorkflow;
-
-namespace ContinuousTests
-{
-    public class ContinuousLogDownloader
-    {
-        private readonly TestLifecycle lifecycle;
-        private readonly CodexDeployment deployment;
-        private readonly string outputPath;
-        private readonly CancellationToken cancelToken;
-
-        public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken)
-        {
-            this.lifecycle = lifecycle;
-            this.deployment = deployment;
-            this.outputPath = outputPath;
-            this.cancelToken = cancelToken;
-        }
-
-        public void Run()
-        {
-            while (!cancelToken.IsCancellationRequested)
-            {
-                UpdateLogs();
-
-                cancelToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(15));
-            }
-
-            // After testing has stopped, we wait a little bit and fetch the logs one more time.
-            // If our latest fetch was not recent, interesting test-related log activity might
-            // not have been captured yet.
-            Thread.Sleep(TimeSpan.FromSeconds(10));
-            UpdateLogs();
-        }
-
-        private void UpdateLogs()
-        {
-            foreach (var container in deployment.CodexContainers)
-            {
-                UpdateLog(container);
-            }
-        }
-
-        private void UpdateLog(RunningContainer container)
-        {
-            var filepath = Path.Combine(outputPath, GetLogName(container));
-            if (!File.Exists(filepath))
-            {
-                File.WriteAllLines(filepath, new[] { container.Name });
-            }
-
-            var appender = new LogAppender(filepath);
-
-            lifecycle.CodexStarter.DownloadLog(container, appender);
-        }
-
-        private static string GetLogName(RunningContainer container)
-        {
-            return container.Name
-                .Replace("<", "")
-                .Replace(">", "")
-                + ".log";
-        }
-    }
-
-    public class LogAppender : ILogHandler
-    {
-        private readonly string filename;
-
-        public LogAppender(string filename)
-        {
-            this.filename = filename;
-        }
-
-        public void Log(Stream log)
-        {
-            using var reader = new StreamReader(log);
-            var lines = File.ReadAllLines(filename);
-            var lastLine = lines.Last();
-            var recording = lines.Length < 3;
-            var line = reader.ReadLine();
-            while (line != null)
-            {
-                if (recording)
-                {
-                    File.AppendAllLines(filename, new[] { line });
-                }
-                else
-                {
-                    recording = line == lastLine;
-                }
-
-                line = reader.ReadLine();
-            }
-        }
-    }
-}

ContinuousTests/ContinuousTest.cs

@@ -1,5 +1,7 @@
 using DistTestCore;
 using DistTestCore.Codex;
+using DistTestCore.Logs;
+using KubernetesWorkflow;
 using Logging;
 
 namespace ContinuousTests
@@ -87,6 +89,12 @@ namespace ContinuousTests
             return file;
         }
 
+        public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null)
+        {
+            var nodeRunner = new NodeRunner(Nodes, Configuration, TimeSet, Log, Configuration.CodexDeployment.Metadata.KubeNamespace, EthereumAccountIndex);
+            return nodeRunner.DownloadLog(container, tailLines);
+        }
+
         private void DownloadToFile(CodexAccess node, string contentId, TestFile file)
         {
             using var fileStream = File.OpenWrite(file.Filename);

ContinuousTests/ContinuousTestRunner.cs

@@ -30,8 +30,6 @@ namespace ContinuousTests
 
             ClearAllCustomNamespaces(allTests, overviewLog);
 
-            StartLogDownloader(taskFactory);
-
             var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
 
             foreach (var testLoop in testLoops)
@@ -60,21 +58,8 @@ namespace ContinuousTests
             if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
 
             log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
-            var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
-            workflowCreator.CreateWorkflow().DeleteTestResources();
-        }
-
-        private void StartLogDownloader(TaskFactory taskFactory)
-        {
-            if (!config.DownloadContainerLogs) return;
-
-            var path = Path.Combine(config.LogPath, "containers");
-            if (!Directory.Exists(path)) Directory.CreateDirectory(path);
-
-            var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
-            var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
-
-            taskFactory.Run(downloader.Run);
+            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log);
+            lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources();
         }
     }
 }

ContinuousTests/K8sFactory.cs

@@ -1,13 +1,12 @@
 using DistTestCore.Codex;
 using DistTestCore;
-using KubernetesWorkflow;
 using Logging;
 
 namespace ContinuousTests
 {
     public class K8sFactory
     {
-        public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation)
+        public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log)
         {
             var kubeConfig = GetKubeConfig(kubeConfigFile);
             var lifecycleConfig = new DistTestCore.Configuration
@@ -17,19 +16,10 @@ namespace ContinuousTests
                 logDebug: false,
                 dataFilesPath: dataFilePath,
                 codexLogLevel: CodexLogLevel.Debug,
-                runnerLocation: runnerLocation
+                k8sNamespacePrefix: customNamespace
             );
 
-            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: customNamespace,
-                kubeConfigFile: kubeConfig,
-                operationTimeout: timeSet.K8sOperationTimeout(),
-            retryDelay: timeSet.WaitForK8sServiceDelay());
-
-            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
-            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator);
-
-            return (workflowCreator, lifecycle);
+            return new TestLifecycle(log, lifecycleConfig, timeSet, "continuous-tests", string.Empty);
         }
 
         private static string? GetKubeConfig(string kubeConfigFile)

ContinuousTests/NodeRunner.cs

@@ -5,6 +5,7 @@ using KubernetesWorkflow;
 using NUnit.Framework;
 using Logging;
 using Utils;
+using DistTestCore.Logs;
 
 namespace ContinuousTests
 {
@@ -38,10 +39,25 @@ namespace ContinuousTests
             RunNode(bootstrapNode, operation, 0.TestTokens());
         }
 
+        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
+        {
+            var subFile = log.CreateSubfile();
+            var description = container.Name;
+            var handler = new LogDownloadHandler(container, description, subFile);
+
+            log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
+
+            var lifecycle = CreateTestLifecycle();
+            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
+            flow.DownloadContainerLog(container, handler, tailLines);
+
+            return new DownloadedLog(subFile, description);
+        }
+
         public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
         {
-            var (workflowCreator, lifecycle) = CreateFacilities();
-            var flow = workflowCreator.CreateWorkflow();
+            var lifecycle = CreateTestLifecycle();
+            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
 
             try
             {
@@ -89,9 +105,9 @@ namespace ContinuousTests
             }
         }
 
-        private (WorkflowCreator, TestLifecycle) CreateFacilities()
+        private TestLifecycle CreateTestLifecycle()
         {
-            return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation);
+            return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log);
         }
     }
 }

ContinuousTests/SingleTestRun.cs

@@ -138,7 +138,7 @@ namespace ContinuousTests
         private void DownloadClusterLogs()
         {
             var k8sFactory = new K8sFactory();
-            var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+            var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
 
             foreach (var container in config.CodexDeployment.CodexContainers)
             {
@@ -221,7 +221,7 @@ namespace ContinuousTests
         private DistTestCore.Configuration CreateFileManagerConfiguration()
         {
             return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
-                CodexLogLevel.Error, config.RunnerLocation);
+                CodexLogLevel.Error, string.Empty);
         }
     }
 }

ContinuousTests/Tests/HoldMyBeerTest.cs

@@ -0,0 +1,39 @@
+using DistTestCore;
+using NUnit.Framework;
+
+namespace ContinuousTests.Tests
+{
+    public class HoldMyBeerTest : ContinuousTest
+    {
+        public override int RequiredNumberOfNodes => 1;
+        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(5);
+        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
+
+        private ContentId? cid;
+        private TestFile file = null!;
+
+        [TestMoment(t: Zero)]
+        public void UploadTestFile()
+        {
+            var metadata = Configuration.CodexDeployment.Metadata;
+            var maxQuotaUseMb = metadata.StorageQuotaMB / 2;
+            var safeTTL = Math.Max(metadata.BlockTTL, metadata.BlockMI) + 30;
+            var runsPerTtl = Convert.ToInt32(safeTTL / RunTestEvery.TotalSeconds);
+            var filesizePerUploadMb = Math.Min(80, maxQuotaUseMb / runsPerTtl);
+            // This filesize should keep the quota below 50% of the node's max.
+
+            var filesize = filesizePerUploadMb.MB();
+            double codexDefaultBlockSize = 31 * 64 * 33;
+            var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
+            Assert.That(numberOfBlocks, Is.EqualTo(1282));
+
+            file = FileManager.GenerateTestFile(filesize);
+
+            cid = UploadFile(Nodes[0], file);
+            Assert.That(cid, Is.Not.Null);
+
+            var dl = DownloadFile(Nodes[0], cid!);
+            file.AssertIsEqual(dl);
+        }
+    }
+}

ContinuousTests/Tests/TwoClientTest.cs

@@ -6,7 +6,7 @@ namespace ContinuousTests.Tests
     public class TwoClientTest : ContinuousTest
     {
         public override int RequiredNumberOfNodes => 2;
-        public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
+        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(1);
         public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
 
         private ContentId? cid;

ContinuousTests/reports/CodexTestNetReport-July2023.md

@@ -0,0 +1,38 @@
+# Codex Continuous Test-net Report
+Date: 02-08-2023
+
+Report for: 07-2023
+
+
+## Test-net Status
+- Start of month: Offline - faulted
+- End of month: Offline - faulted
+
+(Faulted: Tests fail with such frequency that the information gathered does not justify the cost of leaving the test-net running.)
+
+## Deployment Configuration
+Continous Test-net is deployed to the kubernetes cluster with the following configuration:
+
+5x Codex Nodes:
+- Log-level: Trace
+- Storage quota: 2048 MB
+- Storage sell: 1024 MB
+- Min price: 1024
+- Max collateral: 1024
+- Max duration: 3600000 seconds
+- Block-TTL: 120 seconds
+
+3 of these 5 nodes have:
+- Validator: true
+
+Kubernetes namespace: 'codex-continuous-tests'
+
+## Test Overview
+| Changes        | Test                | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | Status      | Results                                                                              |
+|----------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|--------------------------------------------------------------------------------------|
+| New in 07-2023 | Two-client test     | Every 30 seconds, two nodes are chosen at random. A 10 MB file is generated and uploaded to one. 10 seconds later, it is downloaded from the other. File contents are asserted to be equal.                                                                                                                                                                                                                                                                                                                                                                                    | Faulted     | Test reliably fails after 30 to 45 minutes. Both upload and download failures occur. |
+| New in 07-2023 | Transient-node test | Every 1 minute, a new, transient Codex node is started and bootstrapped against a random node of the test net. A 10 MB file is generated and uploaded to the transient node. The file is then downloaded from a second random test net node and file equality is asserted. After that, the transient node is shut down. 30 seconds later, a new transient Codex node is started and bootstrapped against a third (guaranteed different from first and second) node from the test net. The same file is then downloaded from the new transient node. File equality is asserted. | Not running | Test was not run because the previous more rudamentory test has faulted.             |
+
+## Action Points
+- Codex logs (from these long-running containers) are often incomplete when downloaded after a test failure. A reliable way of maintaining these logs is needed. The logs can quickly explode in size, even for test-nets that run only for a few hours.
+- The distributed testing setup can and has been used to reproduce the failure of the Two-client test in a local environment. Investigation is on-going.

DistTestCore/BaseStarter.cs

@@ -1,18 +1,15 @@
-using KubernetesWorkflow;
-using Logging;
+using Logging;
 
 namespace DistTestCore
 {
     public class BaseStarter
     {
         protected readonly TestLifecycle lifecycle;
-        protected readonly WorkflowCreator workflowCreator;
         private Stopwatch? stopwatch;
 
-        public BaseStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+        public BaseStarter(TestLifecycle lifecycle)
         {
             this.lifecycle = lifecycle;
-            this.workflowCreator = workflowCreator;
         }
 
         protected void LogStart(string msg)

DistTestCore/ByteSize.cs

@@ -38,6 +38,11 @@ namespace DistTestCore
     {
         private const long Kilo = 1024;
 
+        public static ByteSize Bytes(this long i)
+        {
+            return new ByteSize(i);
+        }
+
         public static ByteSize KB(this long i)
         {
             return new ByteSize(i * Kilo);
@@ -58,6 +63,11 @@ namespace DistTestCore
             return (i * Kilo).GB();
         }
 
+        public static ByteSize Bytes(this int i)
+        {
+            return new ByteSize(i);
+        }
+
         public static ByteSize KB(this int i)
         {
             return Convert.ToInt64(i).KB();

DistTestCore/Codex/CodexAccess.cs

@@ -4,10 +4,11 @@ using Utils;
 
 namespace DistTestCore.Codex
 {
-    public class CodexAccess
+    public class CodexAccess : ILogHandler
     {
         private readonly BaseLog log;
         private readonly ITimeSet timeSet;
+        private bool hasContainerCrashed;
 
         public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
         {
@@ -15,6 +16,9 @@ namespace DistTestCore.Codex
             Container = container;
             this.timeSet = timeSet;
             Address = address;
+            hasContainerCrashed = false;
+
+            if (container.CrashWatcher != null) container.CrashWatcher.Start(this);
         }
 
         public RunningContainer Container { get; }
@@ -84,9 +88,36 @@ namespace DistTestCore.Codex
             return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
         }
 
+        public string GetName()
+        {
+            return Container.Name;
+        }
+
         private Http Http()
         {
-            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name);
+            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
+        }
+
+        private void CheckContainerCrashed(HttpClient client)
+        {
+            if (hasContainerCrashed) throw new Exception("Container has crashed.");
+        }
+
+        public void Log(Stream crashLog)
+        {
+            var file = log.CreateSubfile();
+            log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
+
+            using var reader = new StreamReader(crashLog);
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                file.Write(line);
+                line = reader.ReadLine();
+            }
+
+            log.Log("Crash log successfully downloaded.");
+            hasContainerCrashed = true;
         }
     }
 }

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -5,7 +5,7 @@ namespace DistTestCore.Codex
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:latest";
+        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
 
         public const string MetricsPortTag = "metrics_port";
         public const string DiscoveryPortTag = "discovery-port";
@@ -14,6 +14,7 @@ namespace DistTestCore.Codex
         public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
         public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
 
+        public override string AppName => "codex";
         public override string Image { get; }
 
         public CodexContainerRecipe()
@@ -50,7 +51,15 @@ namespace DistTestCore.Codex
             {
                 AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
             }
-            if (config.MetricsEnabled)
+            if (config.BlockMaintenanceInterval != null)
+            {
+                AddEnvVar("CODEX_BLOCK_MI", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
+            }
+            if (config.BlockMaintenanceNumber != null)
+            {
+                AddEnvVar("CODEX_BLOCK_MN", config.BlockMaintenanceNumber.ToString()!);
+            }
+            if (config.MetricsMode != Metrics.MetricsMode.None)
             {
                 AddEnvVar("CODEX_METRICS", "true");
                 AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");

DistTestCore/Codex/CodexDeployment.cs

@@ -5,23 +5,25 @@ namespace DistTestCore.Codex
 {
     public class CodexDeployment
     {
-        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata)
+        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, GrafanaStartInfo? grafanaStartInfo, DeploymentMetadata metadata)
         {
             GethStartResult = gethStartResult;
             CodexContainers = codexContainers;
             PrometheusContainer = prometheusContainer;
+            GrafanaStartInfo = grafanaStartInfo;
             Metadata = metadata;
         }
 
         public GethStartResult GethStartResult { get; }
         public RunningContainer[] CodexContainers { get; }
         public RunningContainer? PrometheusContainer { get; }
+        public GrafanaStartInfo? GrafanaStartInfo { get; }
         public DeploymentMetadata Metadata { get; }
     }
 
     public class DeploymentMetadata
     {
-        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration)
+        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
         {
             DeployDateTimeUtc = DateTime.UtcNow;
             KubeNamespace = kubeNamespace;
@@ -33,6 +35,9 @@ namespace DistTestCore.Codex
             MinPrice = minPrice;
             MaxCollateral = maxCollateral;
             MaxDuration = maxDuration;
+            BlockTTL = blockTTL;
+            BlockMI = blockMI;
+            BlockMN = blockMN;
         }
 
         public DateTime DeployDateTimeUtc { get; }
@@ -45,5 +50,8 @@ namespace DistTestCore.Codex
         public int MinPrice { get; }
         public int MaxCollateral { get; }
         public int MaxDuration { get; }
+        public int BlockTTL { get; }
+        public int BlockMI { get; }
+        public int BlockMN { get; }
     }
 }

DistTestCore/Codex/CodexStartupConfig.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
 using KubernetesWorkflow;
 
 namespace DistTestCore.Codex
@@ -14,9 +15,11 @@ namespace DistTestCore.Codex
         public Location Location { get; set; }
         public CodexLogLevel LogLevel { get; }
         public ByteSize? StorageQuota { get; set; }
-        public bool MetricsEnabled { get; set; }
+        public MetricsMode MetricsMode { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
         public string? BootstrapSpr { get; set; }
         public int? BlockTTL { get; set; }
+        public TimeSpan? BlockMaintenanceInterval { get; set; }
+        public int? BlockMaintenanceNumber { get; set; }
     }
 }

DistTestCore/CodexNodeGroup.cs

@@ -14,12 +14,12 @@ namespace DistTestCore
     {
         private readonly TestLifecycle lifecycle;
 
-        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
         {
             this.lifecycle = lifecycle;
             Setup = setup;
             Containers = containers;
-            Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
+            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
             Version = new CodexDebugVersionResponse();
         }
 
@@ -45,7 +45,7 @@ namespace DistTestCore
         }
 
         public CodexSetup Setup { get; private set; }
-        public RunningContainers Containers { get; private set; }
+        public RunningContainers[] Containers { get; private set; }
         public OnlineCodexNode[] Nodes { get; private set; }
         public CodexDebugVersionResponse Version { get; private set; }
 

DistTestCore/CodexSetup.cs

@@ -10,6 +10,9 @@ namespace DistTestCore
         ICodexSetup At(Location location);
         ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
         ICodexSetup WithStorageQuota(ByteSize storageQuota);
+        ICodexSetup WithBlockTTL(TimeSpan duration);
+        ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration);
+        ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks);
         ICodexSetup EnableMetrics();
         ICodexSetup EnableMarketplace(TestToken initialBalance);
         ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
@@ -50,9 +53,27 @@ namespace DistTestCore
             return this;
         }
 
+        public ICodexSetup WithBlockTTL(TimeSpan duration)
+        {
+            BlockTTL = Convert.ToInt32(duration.TotalSeconds);
+            return this;
+        }
+
+        public ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration)
+        {
+            BlockMaintenanceInterval = duration;
+            return this;
+        }
+
+        public ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks)
+        {
+            BlockMaintenanceNumber = numberOfBlocks;
+            return this;
+        }
+
         public ICodexSetup EnableMetrics()
         {
-            MetricsEnabled = true;
+            MetricsMode = Metrics.MetricsMode.Record;
             return this;
         }
 

DistTestCore/CodexStarter.cs

@@ -8,8 +8,8 @@ namespace DistTestCore
 {
     public class CodexStarter : BaseStarter
     {
-        public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public CodexStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
@@ -32,12 +32,12 @@ namespace DistTestCore
             var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
             lifecycle.SetCodexVersion(group.Version);
 
-            var podInfo = group.Containers.RunningPod.PodInfo;
+            var nl = Environment.NewLine;
+            var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
             LogEnd($"Started {codexSetup.NumberOfNodes} nodes " +
-                $"of image '{containers.Containers.First().Recipe.Image}' " +
-                $"and version '{group.Version}' " +
-                $"at location '{podInfo.K8SNodeName}'={podInfo.Ip}. " +
-                $"They are: {group.Describe()}");
+                $"of image '{containers.Containers().First().Recipe.Image}' " +
+                $"and version '{group.Version}'{nl}" +
+                podInfos);
             LogSeparator();
 
             return group;
@@ -47,7 +47,11 @@ namespace DistTestCore
         {
             LogStart($"Stopping {group.Describe()}...");
             var workflow = CreateWorkflow();
-            workflow.Stop(group.Containers);
+            foreach (var c in group.Containers)
+            {
+                StopCrashWatcher(c);
+                workflow.Stop(c);
+            }
             RunningGroups.Remove(group);
             LogEnd("Stopped.");
         }
@@ -60,17 +64,23 @@ namespace DistTestCore
             RunningGroups.Clear();
         }
 
-        public void DownloadLog(RunningContainer container, ILogHandler logHandler)
+        public void DownloadLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
         {
             var workflow = CreateWorkflow();
-            workflow.DownloadContainerLog(container, logHandler);
+            workflow.DownloadContainerLog(container, logHandler, tailLines);
         }
 
-        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers containers)
+        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
         {
-            if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
+            if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory();
 
             var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
+
+            if (codexSetup.MetricsMode == MetricsMode.Dashboard)
+            {
+                lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup);
+            }
+
             return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
         }
 
@@ -83,28 +93,66 @@ namespace DistTestCore
             return startupConfig;
         }
 
-        private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
+        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
         {
-            var workflow = CreateWorkflow();
-            return workflow.Start(numberOfNodes, location, new CodexContainerRecipe(), startupConfig);
+            var result = new List<RunningContainers>();
+            var recipe = new CodexContainerRecipe();
+            for (var i = 0; i < numberOfNodes; i++)
+            {
+                var workflow = CreateWorkflow();
+                var rc = workflow.Start(1, location, recipe, startupConfig);
+                CreateCrashWatcher(workflow, rc);
+                result.Add(rc);
+            }
+            return result.ToArray();
         }
 
-        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory)
+        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
             RunningGroups.Add(group);
-            Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
+
+            try
+            {
+                Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
+            }
+            catch
+            {
+                CodexNodesNotOnline(runningContainers);
+                throw;
+            }
+
             return group;
         }
 
+        private void CodexNodesNotOnline(RunningContainers[] runningContainers)
+        {
+            Log("Codex nodes failed to start");
+            foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
+        }
+
         private StartupWorkflow CreateWorkflow()
         {
-            return workflowCreator.CreateWorkflow();
+            return lifecycle.WorkflowCreator.CreateWorkflow();
         }
 
         private void LogSeparator()
         {
             Log("----------------------------------------------------------------------------");
         }
+
+        private void CreateCrashWatcher(StartupWorkflow workflow, RunningContainers rc)
+        {
+            var c = rc.Containers.Single();
+            c.CrashWatcher = workflow.CreateCrashWatcher(c);
+        }
+
+        private void StopCrashWatcher(RunningContainers containers)
+        {
+            foreach (var c in containers.Containers)
+            {
+                c.CrashWatcher?.Stop();
+            }
+        }
     }
 }

DistTestCore/Configuration.cs

@@ -1,5 +1,6 @@
 using DistTestCore.Codex;
 using KubernetesWorkflow;
+using System.Net.NetworkInformation;
 using Utils;
 
 namespace DistTestCore
@@ -11,7 +12,8 @@ namespace DistTestCore
         private readonly bool logDebug;
         private readonly string dataFilesPath;
         private readonly CodexLogLevel codexLogLevel;
-        private readonly TestRunnerLocation runnerLocation;
+        private readonly string k8sNamespacePrefix;
+        private static RunnerLocation? runnerLocation = null;
 
         public Configuration()
         {
@@ -20,23 +22,23 @@ namespace DistTestCore
             logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
             dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
             codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
-            runnerLocation = ParseEnum.Parse<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
+            k8sNamespacePrefix = "ct-";
         }
 
-        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation)
+        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, string k8sNamespacePrefix)
         {
             this.kubeConfigFile = kubeConfigFile;
             this.logPath = logPath;
             this.logDebug = logDebug;
             this.dataFilesPath = dataFilesPath;
             this.codexLogLevel = codexLogLevel;
-            this.runnerLocation = runnerLocation;
+            this.k8sNamespacePrefix = k8sNamespacePrefix;
         }
 
         public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
         {
             return new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: "ct-",
+                k8sNamespacePrefix: k8sNamespacePrefix,
                 kubeConfigFile: kubeConfigFile,
                 operationTimeout: timeSet.K8sOperationTimeout(),
                 retryDelay: timeSet.WaitForK8sServiceDelay()
@@ -58,14 +60,14 @@ namespace DistTestCore
             return codexLogLevel;
         }
 
-        public TestRunnerLocation GetTestRunnerLocation()
-        {
-            return runnerLocation;
-        }
-
         public Address GetAddress(RunningContainer container)
         {
-            if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster)
+            if (runnerLocation == null)
+            {
+                runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container);
+            }
+            
+            if (runnerLocation == RunnerLocation.InternalToCluster)
             {
                 return container.ClusterInternalAddress;
             }
@@ -87,9 +89,56 @@ namespace DistTestCore
         }
     }
 
-    public enum TestRunnerLocation
+    public enum RunnerLocation
     {
         ExternalToCluster,
         InternalToCluster,
     }
+
+    public static class RunnerLocationUtils
+    {
+        private static bool alreadyDidThat = false;
+
+        public static RunnerLocation DetermineRunnerLocation(RunningContainer container)
+        {
+            // We want to be sure we don't ping more often than strictly necessary.
+            // If we have already determined the location during this application
+            // lifetime, don't do it again.
+            if (alreadyDidThat) throw new Exception("We already did that.");
+            alreadyDidThat = true;
+
+            if (PingHost(container.Pod.PodInfo.Ip))
+            {
+                return RunnerLocation.InternalToCluster;
+            }
+            if (PingHost(Format(container.ClusterExternalAddress)))
+            {
+                return RunnerLocation.ExternalToCluster;
+            }
+
+            throw new Exception("Unable to determine runner location.");
+        }
+
+        private static string Format(Address host)
+        {
+            return host.Host
+                .Replace("http://", "")
+                .Replace("https://", "");
+        }
+
+        private static bool PingHost(string host)
+        {
+            try
+            {
+                using var pinger = new Ping();
+                PingReply reply = pinger.Send(host);
+                return reply.Status == IPStatus.Success;
+            }
+            catch (PingException)
+            {
+            }
+
+            return false;
+        }
+    }
 }

DistTestCore/DistTest.cs

@@ -13,6 +13,7 @@ namespace DistTestCore
     [Parallelizable(ParallelScope.All)]
     public abstract class DistTest
     {
+        private const string TestsType = "dist-tests";
         private readonly Configuration configuration = new Configuration();
         private readonly Assembly[] testAssemblies;
         private readonly FixtureLog fixtureLog;
@@ -29,14 +30,8 @@ namespace DistTestCore
             var startTime = DateTime.UtcNow;
             fixtureLog = new FixtureLog(logConfig, startTime);
             statusLog = new StatusLog(logConfig, startTime);
-
-            PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
-            PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
         }
 
-        public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; }
-        public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; }
-
         [OneTimeSetUp]
         public void GlobalSetup()
         {
@@ -52,7 +47,7 @@ namespace DistTestCore
             {
                 Stopwatch.Measure(fixtureLog, "Global setup", () =>
                 {
-                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
+                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), new PodLabels(TestsType, null!), string.Empty);
                     wc.CreateWorkflow().DeleteAllResources();
                 });                
             }
@@ -174,6 +169,21 @@ namespace DistTestCore
             GetTestLog().Debug(msg);
         }
 
+        public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers()
+        {
+            return new PeerConnectionTestHelpers(GetTestLog());
+        }
+
+        public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers()
+        {
+            return new PeerDownloadTestHelpers(GetTestLog(), Get().FileManager);
+        }
+
+        public void Measure(string name, Action action)
+        {
+            Stopwatch.Measure(Get().Log, name, action);
+        }
+
         protected CodexSetup CreateCodexSetup(int numberOfNodes)
         {
             return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
@@ -195,7 +205,9 @@ namespace DistTestCore
             {
                 lock (lifecycleLock)
                 {
-                    lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
+                    var testNamespace = Guid.NewGuid().ToString();
+                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), TestsType, testNamespace);
+                    lifecycles.Add(testName, lifecycle);
                 }
             });
         }
@@ -206,7 +218,7 @@ namespace DistTestCore
             var testResult = GetTestResult();
             var testDuration = lifecycle.GetTestDuration();
             fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
-            statusLog.ConcludeTest(testResult, testDuration, GetCodexId(lifecycle));
+            statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetApplicationIds());
             Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
             {
                 lifecycle.Log.EndTest();
@@ -216,14 +228,6 @@ namespace DistTestCore
             });
         }
 
-        private static string GetCodexId(TestLifecycle lifecycle)
-        {
-            var v = lifecycle.CodexVersion;
-            if (v == null) return new CodexContainerRecipe().Image;
-            if (v.version != "untagged build") return v.version;
-            return v.revision;
-        }
-
         private ITimeSet GetTimeSet()
         {
             if (ShouldUseLongTimeouts()) return new LongTimeSet();

DistTestCore/DistTestCore.csproj

@@ -10,6 +10,14 @@
   <PropertyGroup Condition="'$(IsArm64)'=='true'">
     <DefineConstants>Arm64</DefineConstants>
   </PropertyGroup>
+  <ItemGroup>
+    <None Remove="Metrics\dashboard.json" />
+  </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="Metrics\dashboard.json">
+      <CopyToOutputDirectory>Never</CopyToOutputDirectory>
+    </EmbeddedResource>
+  </ItemGroup>
 
   <ItemGroup>
 	  <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />

DistTestCore/GethStarter.cs

@@ -1,5 +1,4 @@
 using DistTestCore.Marketplace;
-using KubernetesWorkflow;
 
 namespace DistTestCore
 {
@@ -8,13 +7,13 @@ namespace DistTestCore
         private readonly MarketplaceNetworkCache marketplaceNetworkCache;
         private readonly GethCompanionNodeStarter companionNodeStarter;
 
-        public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public GethStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
             marketplaceNetworkCache = new MarketplaceNetworkCache(
-                new GethBootstrapNodeStarter(lifecycle, workflowCreator),
-                new CodexContractsStarter(lifecycle, workflowCreator));
-            companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator);
+                new GethBootstrapNodeStarter(lifecycle),
+                new CodexContractsStarter(lifecycle));
+            companionNodeStarter = new GethCompanionNodeStarter(lifecycle);
         }
 
         public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup)

DistTestCore/GrafanaStarter.cs

@@ -0,0 +1,192 @@
+using DistTestCore.Metrics;
+using IdentityModel.Client;
+using KubernetesWorkflow;
+using Newtonsoft.Json;
+using System.Reflection;
+
+namespace DistTestCore
+{
+    public class GrafanaStarter : BaseStarter
+    {
+        private const string StorageQuotaThresholdReplaceToken = "\"<CODEX_STORAGEQUOTA>\"";
+        private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"<CODEX_BYTESUSED_SOFTMAX>\"";
+
+        public GrafanaStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
+        {
+        }
+
+        public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup)
+        {
+            LogStart($"Starting dashboard server");
+
+            var grafanaContainer = StartGrafanaContainer();
+            var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer);
+
+            var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth);
+
+            Log("Connecting datasource...");
+            AddDataSource(http, prometheusContainer);
+
+            Log("Uploading dashboard configurations...");
+            var jsons = ReadEachDashboardJsonFile(codexSetup);
+            var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
+
+            LogEnd("Dashboard server started.");
+
+            return new GrafanaStartInfo(dashboardUrls, grafanaContainer);
+        }
+
+        private RunningContainer StartGrafanaContainer()
+        {
+            var startupConfig = new StartupConfig();
+
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig);
+            if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created.");
+
+            return grafanaContainers.Containers.First();
+        }
+
+        private void AddBasicAuth(HttpClient client)
+        {
+            client.SetBasicAuthentication(
+                GrafanaContainerRecipe.DefaultAdminUser,
+                GrafanaContainerRecipe.DefaultAdminPassword);
+        }
+
+        private static void AddDataSource(Http http, RunningContainer prometheusContainer)
+        {
+            var prometheusAddress = prometheusContainer.ClusterExternalAddress;
+            var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port;
+            var response = http.HttpPostJson<GrafanaDataSourceRequest, GrafanaDataSourceResponse>("datasources", new GrafanaDataSourceRequest
+            {
+                uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb",
+                name = "CodexPrometheus",
+                type = "prometheus",
+                url = prometheusUrl,
+                access = "proxy",
+                basicAuth = false,
+                jsonData = new GrafanaDataSourceJsonData
+                {
+                    httpMethod = "POST"
+                }
+            });
+
+            if (response.message != "Datasource added")
+            {
+                throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message);
+            }
+        }
+
+        public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson)
+        {
+            var request = GetDashboardCreateRequest(dashboardJson);
+            var response = http.HttpPostString("dashboards/db", request);
+            var jsonResponse = JsonConvert.DeserializeObject<GrafanaPostDashboardResponse>(response);
+            if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard.");
+
+            var grafanaAddress = grafanaContainer.ClusterExternalAddress;
+            return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
+        }
+
+        private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup)
+        {
+            var assembly = Assembly.GetExecutingAssembly();
+            var resourceNames = new[]
+            {
+                "DistTestCore.Metrics.dashboard.json"
+            };
+
+            return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray();
+        }
+
+        private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup)
+        {
+            using var stream = assembly.GetManifestResourceStream(resourceName);
+            if (stream == null) throw new Exception("Unable to find resource " + resourceName);
+            using var reader = new StreamReader(stream);
+            return ApplyReplacements(reader.ReadToEnd(), codexSetup);
+        }
+
+        private static string ApplyReplacements(string input, CodexSetup codexSetup)
+        {
+            var quotaString = GetQuotaString(codexSetup);
+            var softMaxString = GetSoftMaxString(codexSetup);
+
+            return input
+                .Replace(StorageQuotaThresholdReplaceToken, quotaString)
+                .Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString);
+        }
+
+        private static string GetQuotaString(CodexSetup codexSetup)
+        {
+            return GetCodexStorageQuotaInBytes(codexSetup).ToString();
+        }
+
+        private static string GetSoftMaxString(CodexSetup codexSetup)
+        {
+            var quota = GetCodexStorageQuotaInBytes(codexSetup);
+            var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing.
+            return softMax.ToString();
+        }
+
+        private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup)
+        {
+            if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes;
+
+            // Codex default: 8GB
+            return 8.GB().SizeInBytes;
+        }
+
+        private static string GetDashboardCreateRequest(string dashboardJson)
+        {
+            return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}";
+        }
+    }
+
+    public class GrafanaStartInfo
+    {
+        public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container)
+        {
+            DashboardUrls = dashboardUrls;
+            Container = container;
+        }
+
+        public string[] DashboardUrls { get; }
+        public RunningContainer Container { get; }
+    }
+
+    public class GrafanaDataSourceRequest
+    {
+        public string uid { get; set; } = string.Empty;
+        public string name { get; set; } = string.Empty;
+        public string type { get; set; } = string.Empty;
+        public string url { get; set; } = string.Empty;
+        public string access { get; set; } = string.Empty;
+        public bool basicAuth { get; set; }
+        public GrafanaDataSourceJsonData jsonData { get; set; } = new();
+    }
+
+    public class GrafanaDataSourceResponse
+    {
+        public int id { get; set; }
+        public string message { get; set; } = string.Empty;
+        public string name { get; set; } = string.Empty;
+    }
+
+    public class GrafanaDataSourceJsonData
+    {
+        public string httpMethod { get; set; } = string.Empty;
+    }
+
+    public class GrafanaPostDashboardResponse
+    {
+        public int id { get; set; }
+        public string slug { get; set; } = string.Empty;
+        public string status { get; set; } = string.Empty;
+        public string uid { get; set; } = string.Empty;
+        public string url { get; set; } = string.Empty;
+        public int version { get; set; }
+    }
+}

DistTestCore/Helpers/FullConnectivityHelper.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Codex;
+using Logging;
 using NUnit.Framework;
 using Utils;
 
@@ -14,23 +15,23 @@ namespace DistTestCore.Helpers
     public class FullConnectivityHelper
     {
         private static string Nl = Environment.NewLine;
-        private readonly DistTest test;
+        private readonly BaseLog log;
         private readonly IFullConnectivityImplementation implementation;
 
-        public FullConnectivityHelper(DistTest test, IFullConnectivityImplementation implementation)
+        public FullConnectivityHelper(BaseLog log, IFullConnectivityImplementation implementation)
         {
-            this.test = test;
+            this.log = log;
             this.implementation = implementation;
         }
 
-        public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
+        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
         {
             AssertFullyConnected(nodes.ToArray());
         }
 
-        private void AssertFullyConnected(IOnlineCodexNode[] nodes)
+        private void AssertFullyConnected(CodexAccess[] nodes)
         {
-            test.Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
             var entries = CreateEntries(nodes);
             var pairs = CreatePairs(entries);
 
@@ -43,19 +44,19 @@ namespace DistTestCore.Helpers
             {
                 var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
 
-                test.Log($"Connections failed:{Nl}{pairDetails}");
+                Log($"Connections failed:{Nl}{pairDetails}");
 
                 Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
             }
             else
             {
-                test.Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
+                Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
             }
         }
 
         private static void RetryWhilePairs(List<Pair> pairs, Action action)
         {
-            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2);
             while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
             {
                 action();
@@ -72,7 +73,7 @@ namespace DistTestCore.Helpers
 
             foreach (var pair in selectedPair)
             {
-                test.ScopedTestFiles(pair.Check);
+                pair.Check();
 
                 if (pair.Success)
                 {
@@ -81,10 +82,10 @@ namespace DistTestCore.Helpers
                 }
             }
 
-            test.Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
+            Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
         }
 
-        private Entry[] CreateEntries(IOnlineCodexNode[] nodes)
+        private Entry[] CreateEntries(CodexAccess[] nodes)
         {
             var entries = nodes.Select(n => new Entry(n)).ToArray();
 
@@ -117,15 +118,20 @@ namespace DistTestCore.Helpers
             }
         }
 
+        private void Log(string msg)
+        {
+            log.Log(msg);
+        }
+
         public class Entry
         {
-            public Entry(IOnlineCodexNode node)
+            public Entry(CodexAccess node)
             {
                 Node = node;
                 Response = node.GetDebugInfo();
             }
 
-            public IOnlineCodexNode Node { get; }
+            public CodexAccess Node { get; }
             public CodexDebugResponse Response { get; }
 
             public override string ToString()

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Codex;
+using Logging;
 using static DistTestCore.Helpers.FullConnectivityHelper;
 
 namespace DistTestCore.Helpers
@@ -7,12 +8,17 @@ namespace DistTestCore.Helpers
     {
         private readonly FullConnectivityHelper helper;
 
-        public PeerConnectionTestHelpers(DistTest test)
+        public PeerConnectionTestHelpers(BaseLog log)
         {
-            helper = new FullConnectivityHelper(test, this);
+            helper = new FullConnectivityHelper(log, this);
         }
 
         public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullyConnected(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess));
+        }
+
+        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
         {
             helper.AssertFullyConnected(nodes);
         }
@@ -57,9 +63,8 @@ namespace DistTestCore.Helpers
             var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
             if (peer == null) return $"peerId: {node.peerId} is not known.";
 
-            var n = (OnlineCodexNode)peer.Node;
-            var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
-            var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
+            var ip = peer.Node.Container.Pod.PodInfo.Ip;
+            var discPort = peer.Node.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
             return $"{ip}:{discPort.Number}";
         }
     }

DistTestCore/Helpers/PeerDownloadTestHelpers.cs

@@ -1,21 +1,30 @@
-using static DistTestCore.Helpers.FullConnectivityHelper;
+using DistTestCore.Codex;
+using Logging;
+using static DistTestCore.Helpers.FullConnectivityHelper;
 
 namespace DistTestCore.Helpers
 {
     public class PeerDownloadTestHelpers : IFullConnectivityImplementation
     {
         private readonly FullConnectivityHelper helper;
-        private readonly DistTest test;
+        private readonly BaseLog log;
+        private readonly FileManager fileManager;
         private ByteSize testFileSize;
 
-        public PeerDownloadTestHelpers(DistTest test)
+        public PeerDownloadTestHelpers(BaseLog log, FileManager fileManager)
         {
-            helper = new FullConnectivityHelper(test, this);
+            helper = new FullConnectivityHelper(log, this);
             testFileSize = 1.MB();
-            this.test = test;
+            this.log = log;
+            this.fileManager = fileManager;
         }
 
         public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
+        {
+            AssertFullDownloadInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess), testFileSize);
+        }
+
+        public void AssertFullDownloadInterconnectivity(IEnumerable<CodexAccess> nodes, ByteSize testFileSize)
         {
             this.testFileSize = testFileSize;
             helper.AssertFullyConnected(nodes);
@@ -33,13 +42,15 @@ namespace DistTestCore.Helpers
 
         public PeerConnectionState Check(Entry from, Entry to)
         {
+            fileManager.PushFileSet();
             var expectedFile = GenerateTestFile(from.Node, to.Node);
 
-            var contentId = from.Node.UploadFile(expectedFile);
+            using var uploadStream = File.OpenRead(expectedFile.Filename);
+            var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(uploadStream));
 
             try
             {
-                var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded");
+                var downloadedFile = Stopwatch.Measure(log, "Download", () => DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded"));
                 expectedFile.AssertIsEqual(downloadedFile);
                 return PeerConnectionState.Connection;
             }
@@ -49,16 +60,29 @@ namespace DistTestCore.Helpers
                 // We consider that as no-connection for the purpose of this test.
                 return PeerConnectionState.NoConnection;
             }
+            finally
+            {
+                fileManager.PopFileSet();
+            }
 
             // Should an exception occur during upload, then this try is inconclusive and we try again next loop.
         }
 
-        private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader)
+        private TestFile DownloadFile(CodexAccess node, string contentId, string label)
+        {
+            var downloadedFile = fileManager.CreateEmptyTestFile(label);
+            using var downloadStream = File.OpenWrite(downloadedFile.Filename);
+            using var stream = node.DownloadFile(contentId);
+            stream.CopyTo(downloadStream);
+            return downloadedFile;
+        }
+
+        private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader)
         {
             var up = uploader.GetName().Replace("<", "").Replace(">", "");
             var down = downloader.GetName().Replace("<", "").Replace(">", "");
             var label = $"~from:{up}-to:{down}~";
-            return test.GenerateTestFile(testFileSize, label);
+            return fileManager.GenerateTestFile(testFileSize, label);
         }
     }
 }

DistTestCore/Http.cs

@@ -12,14 +12,21 @@ namespace DistTestCore
         private readonly ITimeSet timeSet;
         private readonly Address address;
         private readonly string baseUrl;
+        private readonly Action<HttpClient> onClientCreated;
         private readonly string? logAlias;
 
         public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
+            : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
+        {
+        }
+
+        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
         {
             this.log = log;
             this.timeSet = timeSet;
             this.address = address;
             this.baseUrl = baseUrl;
+            this.onClientCreated = onClientCreated;
             this.logAlias = logAlias;
             if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
             if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
@@ -66,6 +73,22 @@ namespace DistTestCore
             }, $"HTTP-POST-JSON: {route}");
         }
 
+        public string HttpPostString(string route, string body)
+        {
+            return Retry(() =>
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                Log(url, body);
+                var content = new StringContent(body);
+                content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+                var result = Time.Wait(client.PostAsync(url, content));
+                var str = Time.Wait(result.Content.ReadAsStringAsync());
+                Log(url, str);
+                return str;
+            }, $"HTTP-POST-STRING: {route}");
+        }
+
         public string HttpPostStream(string route, Stream stream)
         {
             return Retry(() =>
@@ -76,7 +99,7 @@ namespace DistTestCore
                 var content = new StreamContent(stream);
                 content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
                 var response = Time.Wait(client.PostAsync(url, content));
-                var str =Time.Wait(response.Content.ReadAsStringAsync());
+                var str = Time.Wait(response.Content.ReadAsStringAsync());
                 Log(url, str);
                 return str;
             }, $"HTTP-POST-STREAM: {route}");
@@ -132,7 +155,12 @@ namespace DistTestCore
         {
             var client = new HttpClient();
             client.Timeout = timeSet.HttpCallTimeout();
+            onClientCreated(client);
             return client;
         }
+
+        private static void DoNothing(HttpClient client)
+        {
+        }
     }
 }

DistTestCore/Logs/DownloadedLog.cs

@@ -6,6 +6,8 @@ namespace DistTestCore.Logs
     public interface IDownloadedLog
     {
         void AssertLogContains(string expectedString);
+        string[] FindLinesThatContain(params string[] tags);
+        void DeleteFile();
     }
 
     public class DownloadedLog : IDownloadedLog
@@ -33,5 +35,30 @@ namespace DistTestCore.Logs
 
             Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
         }
+
+        public string[] FindLinesThatContain(params string[] tags)
+        {
+            var result = new List<string>();
+            using var file = File.OpenRead(logFile.FullFilename);
+            using var streamReader = new StreamReader(file);
+
+            var line = streamReader.ReadLine();
+            while (line != null)
+            {
+                if (tags.All(line.Contains))
+                {
+                    result.Add(line);
+                }
+
+                line = streamReader.ReadLine();
+            }
+
+            return result.ToArray();
+        }
+
+        public void DeleteFile()
+        {
+            File.Delete(logFile.FullFilename);
+        }
     }
 }

DistTestCore/Marketplace/CodexContractsContainerRecipe.cs

@@ -7,12 +7,8 @@ namespace DistTestCore.Marketplace
         public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
         public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
 
-        public override string Image { get; }
-
-        public CodexContractsContainerRecipe()
-        {
-            Image = "codexstorage/dist-tests-codex-contracts-eth:sha-b4e4897";
-        }
+        public override string AppName => "codex-contracts";
+        public override string Image => "codexstorage/codex-contracts-eth:latest-dist-tests";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -6,8 +6,8 @@ namespace DistTestCore.Marketplace
     public class CodexContractsStarter : BaseStarter
     {
 
-        public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public CodexContractsStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
@@ -15,7 +15,7 @@ namespace DistTestCore.Marketplace
         {
             LogStart("Deploying Codex Marketplace...");
 
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]);
 
             var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig);
@@ -25,7 +25,7 @@ namespace DistTestCore.Marketplace
             WaitUntil(() =>
             {
                 var logHandler = new ContractsReadyLogHandler(Debug);
-                workflow.DownloadContainerLog(container, logHandler);
+                workflow.DownloadContainerLog(container, logHandler, null);
                 return logHandler.Found;
             });
             Log("Contracts deployed. Extracting addresses...");
@@ -91,8 +91,13 @@ namespace DistTestCore.Marketplace
         {
             debug(line);
             if (line.Contains(RequiredCompiledString)) SeenCompileString = true;
+            if (line.Contains(ReadyString))
+            {
+                if (!SeenCompileString) throw new Exception("CodexContracts deployment failed. " +
+                    "Solidity files not compiled before process exited.");
 
-            if (SeenCompileString && line.Contains(ReadyString)) Found = true;
+                Found = true;
+            }
         }
     }
 }

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -80,7 +80,7 @@ namespace DistTestCore.Marketplace
         private string FetchPubKey()
         {
             var enodeFinder = new PubKeyFinder(s => log.Debug(s));
-            workflow.DownloadContainerLog(container, enodeFinder);
+            workflow.DownloadContainerLog(container, enodeFinder, null);
             return enodeFinder.GetPubKey();
         }
 

DistTestCore/Marketplace/GethBootstrapNodeStarter.cs

@@ -4,8 +4,8 @@ namespace DistTestCore.Marketplace
 {
     public class GethBootstrapNodeStarter : BaseStarter
     {
-        public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public GethBootstrapNodeStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
@@ -14,7 +14,7 @@ namespace DistTestCore.Marketplace
             LogStart("Starting Geth bootstrap node...");
             var startupConfig = CreateBootstrapStartupConfig();
             
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
             var bootstrapContainer = containers.Containers[0];

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -7,8 +7,8 @@ namespace DistTestCore.Marketplace
     {
         private int companionAccountIndex = 0;
 
-        public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public GethCompanionNodeStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
@@ -18,7 +18,7 @@ namespace DistTestCore.Marketplace
 
             var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
 
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
             var container = containers.Containers[0];

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -10,12 +10,8 @@ namespace DistTestCore.Marketplace
         public const string DiscoveryPortTag = "disc_port";
         public const string AccountsFilename = "accounts.csv";
 
-        public override string Image { get; }
-
-        public GethContainerRecipe()
-        {
-            Image = "codexstorage/dist-tests-geth:sha-b788a2d";
-        }
+        public override string AppName => "geth";
+        public override string Image => "codexstorage/dist-tests-geth:latest";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

DistTestCore/Metrics/GrafanaContainerRecipe.cs

@@ -0,0 +1,25 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore.Metrics
+{
+    public class GrafanaContainerRecipe : ContainerRecipeFactory
+    {
+        public override string AppName => "grafana";
+        public override string Image => "grafana/grafana-oss:10.0.3";
+
+        public const string DefaultAdminUser = "adminium";
+        public const string DefaultAdminPassword = "passwordium";
+
+        protected override void Initialize(StartupConfig startupConfig)
+        {
+            AddExposedPort(3000);
+
+            AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true");
+            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org.");
+            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor");
+
+            AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser);
+            AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword);
+        }
+    }
+}

DistTestCore/Metrics/MetricsMode.cs

@@ -0,0 +1,9 @@
+namespace DistTestCore.Metrics
+{
+    public enum MetricsMode
+    {
+        None,
+        Record,
+        Dashboard
+    }
+}

DistTestCore/Metrics/PrometheusContainerRecipe.cs

@@ -4,12 +4,8 @@ namespace DistTestCore.Metrics
 {
     public class PrometheusContainerRecipe : ContainerRecipeFactory
     {
-        public override string Image { get; }
-
-        public PrometheusContainerRecipe()
-        {
-            Image = "codexstorage/dist-tests-prometheus:sha-f97d7fd";
-        }
+        public override string AppName => "prometheus";
+        public override string Image => "codexstorage/dist-tests-prometheus:latest";
 
         protected override void Initialize(StartupConfig startupConfig)
         {

DistTestCore/OnlineCodexNode.cs

@@ -4,6 +4,7 @@ using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
 using Logging;
 using NUnit.Framework;
+using Utils;
 
 namespace DistTestCore
 {
@@ -15,7 +16,7 @@ namespace DistTestCore
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(IOnlineCodexNode node);
-        IDownloadedLog DownloadLog();
+        IDownloadedLog DownloadLog(int? tailLines = null);
         IMetricsAccess Metrics { get; }
         IMarketplaceAccess Marketplace { get; }
         CodexDebugVersionResponse Version { get; }
@@ -67,6 +68,7 @@ namespace DistTestCore
             using var fileStream = File.OpenRead(file.Filename);
 
             var logMessage = $"Uploading file {file.Describe()}...";
+            Log(logMessage);
             var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
             {
                 return CodexAccess.UploadFile(fileStream);
@@ -82,6 +84,7 @@ namespace DistTestCore
         public TestFile? DownloadContent(ContentId contentId, string fileLabel = "")
         {
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
+            Log(logMessage);
             var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel);
             Stopwatch.Measure(lifecycle.Log, logMessage, () => DownloadToFile(contentId.Id, file));
             Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
@@ -100,9 +103,9 @@ namespace DistTestCore
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public IDownloadedLog DownloadLog()
+        public IDownloadedLog DownloadLog(int? tailLines = null)
         {
-            return lifecycle.DownloadLog(CodexAccess.Container);
+            return lifecycle.DownloadLog(CodexAccess.Container, tailLines);
         }
 
         public ICodexSetup BringOffline()
@@ -116,7 +119,7 @@ namespace DistTestCore
 
         public void EnsureOnlineGetVersionResponse()
         {
-            var debugInfo = CodexAccess.GetDebugInfo();
+            var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
             var nodePeerId = debugInfo.id;
             var nodeName = CodexAccess.Container.Name;
 
@@ -135,14 +138,9 @@ namespace DistTestCore
             var multiAddress = peerInfo.addrs.First();
             // Todo: Is there a case where First address in list is not the way?
 
-            if (Group == peer.Group)
-            {
-                return multiAddress;
-            }
-
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip);
+            return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip);
         }
 
         private void DownloadToFile(string contentId, TestFile file)

DistTestCore/PrometheusStarter.cs

@@ -7,23 +7,21 @@ namespace DistTestCore
 {
     public class PrometheusStarter : BaseStarter
     {
-        public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
-            : base(lifecycle, workflowCreator)
+        public PrometheusStarter(TestLifecycle lifecycle)
+            : base(lifecycle)
         {
         }
 
-        public RunningContainers CollectMetricsFor(RunningContainers containers)
+        public RunningContainers CollectMetricsFor(RunningContainers[] containers)
         {
             LogStart($"Starting metrics server for {containers.Describe()}");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers)));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers())));
 
-            var workflow = workflowCreator.CreateWorkflow();
+            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
             var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 
-            LogEnd("Metrics server started.");
-
             return runningContainers;
         }
 
@@ -31,7 +29,7 @@ namespace DistTestCore
         {
             var config = "";
             config += "global:\n";
-            config += "  scrape_interval: 30s\n";
+            config += "  scrape_interval: 10s\n";
             config += "  scrape_timeout: 10s\n";
             config += "\n";
             config += "scrape_configs:\n";

DistTestCore/TestLifecycle.cs

@@ -1,5 +1,7 @@
 using DistTestCore.Codex;
 using DistTestCore.Logs;
+using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
 using KubernetesWorkflow;
 using Logging;
 using Utils;
@@ -10,21 +12,20 @@ namespace DistTestCore
     {
         private readonly DateTime testStart;
 
-        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet)
-            : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)))
-        {
-        }
-
-        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
+        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testsType, string testNamespace)
         {
             Log = log;
             Configuration = configuration;
             TimeSet = timeSet;
 
+            var podLabels = new PodLabels(testsType, GetApplicationIds());
+            WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), podLabels, testNamespace);
+
             FileManager = new FileManager(Log, configuration);
-            CodexStarter = new CodexStarter(this, workflowCreator);
-            PrometheusStarter = new PrometheusStarter(this, workflowCreator);
-            GethStarter = new GethStarter(this, workflowCreator);
+            CodexStarter = new CodexStarter(this);
+            PrometheusStarter = new PrometheusStarter(this);
+            GrafanaStarter = new GrafanaStarter(this);
+            GethStarter = new GethStarter(this);
             testStart = DateTime.UtcNow;
             CodexVersion = null;
 
@@ -34,9 +35,11 @@ namespace DistTestCore
         public BaseLog Log { get; }
         public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
+        public WorkflowCreator WorkflowCreator { get; }
         public FileManager FileManager { get; }
         public CodexStarter CodexStarter { get; }
         public PrometheusStarter PrometheusStarter { get; }
+        public GrafanaStarter GrafanaStarter { get; }
         public GethStarter GethStarter { get; }
         public CodexDebugVersionResponse? CodexVersion { get; private set; }
 
@@ -46,14 +49,14 @@ namespace DistTestCore
             FileManager.DeleteAllTestFiles();
         }
 
-        public IDownloadedLog DownloadLog(RunningContainer container)
+        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
         {
             var subFile = Log.CreateSubfile();
             var description = container.Name;
             var handler = new LogDownloadHandler(container, description, subFile);
 
             Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-            CodexStarter.DownloadLog(container, handler);
+            CodexStarter.DownloadLog(container, handler, tailLines);
 
             return new DownloadedLog(subFile, description);
         }
@@ -68,5 +71,24 @@ namespace DistTestCore
         {
             if (CodexVersion == null) CodexVersion = version;
         }
+
+        public ApplicationIds GetApplicationIds()
+        {
+            return new ApplicationIds(
+                codexId: GetCodexId(),
+                gethId: new GethContainerRecipe().Image,
+                prometheusId: new PrometheusContainerRecipe().Image,
+                codexContractsId: new CodexContractsContainerRecipe().Image,
+                grafanaId: new GrafanaContainerRecipe().Image
+            );
+        }
+
+        private string GetCodexId()
+        {
+            var v = CodexVersion;
+            if (v == null) return new CodexContainerRecipe().Image;
+            if (v.version != "untagged build") return v.version;
+            return v.revision;
+        }
     }
 }

DistTestCore/Timing.cs

@@ -21,7 +21,7 @@ namespace DistTestCore
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromSeconds(10);
+            return TimeSpan.FromMinutes(5);
         }
 
         public TimeSpan HttpCallRetryTime()
@@ -36,12 +36,12 @@ namespace DistTestCore
 
         public TimeSpan WaitForK8sServiceDelay()
         {
-            return TimeSpan.FromSeconds(1);
+            return TimeSpan.FromSeconds(10);
         }
 
         public TimeSpan K8sOperationTimeout()
         {
-            return TimeSpan.FromMinutes(1);
+            return TimeSpan.FromMinutes(30);
         }
 
         public TimeSpan WaitForMetricTimeout()

KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -27,6 +27,7 @@
             return recipe;
         }
 
+        public abstract string AppName { get; }
         public abstract string Image { get; }
         protected int ContainerNumber { get; private set; } = 0;
         protected int Index { get; private set; } = 0;
@@ -39,6 +40,13 @@
             return p;
         }
 
+        protected Port AddExposedPort(int number, string tag = "")
+        {
+            var p = factory.CreatePort(number, tag);
+            exposedPorts.Add(p);
+            return p;
+        }
+
         protected Port AddInternalPort(string tag = "")
         {
             var p = factory.CreatePort(tag);

KubernetesWorkflow/CrashWatcher.cs

@@ -0,0 +1,95 @@
+using k8s;
+using Logging;
+
+namespace KubernetesWorkflow
+{
+    public class CrashWatcher
+    {
+        private readonly BaseLog log;
+        private readonly KubernetesClientConfiguration config;
+        private readonly string k8sNamespace;
+        private readonly RunningContainer container;
+        private ILogHandler? logHandler;
+        private CancellationTokenSource cts;
+        private Task? worker;
+        private Exception? workerException;
+
+        public CrashWatcher(BaseLog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container)
+        {
+            this.log = log;
+            this.config = config;
+            this.k8sNamespace = k8sNamespace;
+            this.container = container;
+            cts = new CancellationTokenSource();
+        }
+
+        public void Start(ILogHandler logHandler)
+        {
+            if (worker != null) throw new InvalidOperationException();
+
+            this.logHandler = logHandler;
+            cts = new CancellationTokenSource();
+            worker = Task.Run(Worker);
+        }
+
+        public void Stop()
+        {
+            if (worker == null) throw new InvalidOperationException();
+
+            cts.Cancel();
+            worker.Wait();
+            worker = null;
+
+            if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
+        }
+
+        public bool HasContainerCrashed()
+        {
+            using var client = new Kubernetes(config);
+            return HasContainerBeenRestarted(client, container.Pod.PodInfo.Name);
+        }
+
+        private void Worker()
+        {
+            try
+            {
+                MonitorContainer(cts.Token);
+            }
+            catch (Exception ex)
+            {
+                workerException = ex;
+            }
+        }
+
+        private void MonitorContainer(CancellationToken token)
+        {
+            using var client = new Kubernetes(config);
+            while (!token.IsCancellationRequested)
+            {
+                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(1));
+
+                var pod = container.Pod;
+                var recipe = container.Recipe;
+                var podName = pod.PodInfo.Name;
+                if (HasContainerBeenRestarted(client, podName))
+                {
+                    DownloadCrashedContainerLogs(client, podName, recipe);
+                    return;
+                }
+            }
+        }
+
+        private bool HasContainerBeenRestarted(Kubernetes client, string podName)
+        {
+            var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
+            return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
+        }
+
+        private void DownloadCrashedContainerLogs(Kubernetes client, string podName, ContainerRecipe recipe)
+        {
+            log.Log("Pod crash detected for " + container.Name);
+            using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipe.Name, previous: true);
+            logHandler!.Log(stream);
+        }
+    }
+}

KubernetesWorkflow/K8sController.cs

@@ -11,14 +11,16 @@ namespace KubernetesWorkflow
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownPods;
         private readonly WorkflowNumberSource workflowNumberSource;
+        private readonly PodLabels podLabels;
         private readonly K8sClient client;
 
-        public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
+        public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace, PodLabels podLabels)
         {
             this.log = log;
             this.cluster = cluster;
             this.knownPods = knownPods;
             this.workflowNumberSource = workflowNumberSource;
+            this.podLabels = podLabels;
             client = new K8sClient(cluster.GetK8sClientConfig());
 
             K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
@@ -51,10 +53,10 @@ namespace KubernetesWorkflow
             WaitUntilPodOffline(pod.PodInfo.Name);
         }
 
-        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
+        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler, int? tailLines)
         {
             log.Debug();
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, tailLines: tailLines));
             logHandler.Log(stream);
         }
 
@@ -362,7 +364,7 @@ namespace KubernetesWorkflow
 
         private IDictionary<string, string> GetSelector()
         {
-            return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
+            return podLabels.GetLabels();
         }
 
         private IDictionary<string, string> GetRunnerNamespaceSelector()
@@ -602,6 +604,11 @@ namespace KubernetesWorkflow
 
         #endregion
 
+        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        {
+            return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sTestNamespace, container);
+        }
+
         private PodInfo FetchNewPod()
         {
             var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;

KubernetesWorkflow/PodLabels.cs

@@ -0,0 +1,61 @@
+using Logging;
+
+namespace KubernetesWorkflow
+{
+    public class PodLabels
+    {
+        private readonly Dictionary<string, string> labels = new Dictionary<string, string>();
+
+        private PodLabels(PodLabels source)
+        {
+            labels = source.labels.ToDictionary(p => p.Key, p => p.Value);
+        }
+
+        public PodLabels(string testsType, ApplicationIds applicationIds)
+        {
+            Add("tests-type", testsType);
+            Add("runid", NameUtils.GetRunId());
+            Add("testid", NameUtils.GetTestId());
+            Add("category", NameUtils.GetCategoryName());
+            Add("fixturename", NameUtils.GetRawFixtureName());
+            Add("testname", NameUtils.GetTestMethodName());
+
+            if (applicationIds == null) return;
+            Add("codexid", applicationIds.CodexId);
+            Add("gethid", applicationIds.GethId);
+            Add("prometheusid", applicationIds.PrometheusId);
+            Add("codexcontractsid", applicationIds.CodexContractsId);
+            Add("grafanaid", applicationIds.GrafanaId);
+        }
+
+        public PodLabels GetLabelsForAppName(string appName)
+        {
+            var pl = new PodLabels(this);
+            pl.Add("app", appName);
+            return pl;
+        }
+
+        private void Add(string key, string value)
+        {
+            labels.Add(key, Format(value));
+        }
+
+        private static string Format(string s)
+        {
+            var result = s.ToLowerInvariant()
+                .Replace(":", "-")
+                .Replace("/", "-")
+                .Replace("\\", "-")
+                .Replace("[", "-")
+                .Replace("]", "-")
+                .Replace(",", "-");
+
+            return result.Trim('-');
+        }
+
+        internal Dictionary<string, string> GetLabels()
+        {
+            return labels;
+        }
+    }
+}

KubernetesWorkflow/RecipeComponentFactory.cs

@@ -7,6 +7,11 @@ namespace KubernetesWorkflow
     {
         private NumberSource portNumberSource = new NumberSource(8080);
 
+        public Port CreatePort(int number, string tag)
+        {
+            return new Port(number, tag);
+        }
+
         public Port CreatePort(string tag)
         {
             return new Port(portNumberSource.GetNextNumber(), tag);

KubernetesWorkflow/RunningContainers.cs

@@ -1,4 +1,5 @@
-using Utils;
+using Newtonsoft.Json;
+using Utils;
 
 namespace KubernetesWorkflow
 {
@@ -39,5 +40,21 @@ namespace KubernetesWorkflow
         public Port[] ServicePorts { get; }
         public Address ClusterExternalAddress { get; }
         public Address ClusterInternalAddress { get; }
+
+        [JsonIgnore]
+        public CrashWatcher? CrashWatcher { get; set; }
+    }
+
+    public static class RunningContainersExtensions
+    {
+        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
+        {
+            return runningContainers.SelectMany(c => c.Containers).ToArray();
+        }
+
+        public static string Describe(this RunningContainers[] runningContainers)
+        {
+            return string.Join(",", runningContainers.Select(c => c.Describe()));
+        }
     }
 }

KubernetesWorkflow/StartupWorkflow.cs

@@ -10,19 +10,23 @@ namespace KubernetesWorkflow
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownK8SPods;
         private readonly string testNamespace;
+        private readonly PodLabels podLabels;
         private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
 
-        internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
+        internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace, PodLabels podLabels)
         {
             this.log = log;
             this.numberSource = numberSource;
             this.cluster = cluster;
             this.knownK8SPods = knownK8SPods;
             this.testNamespace = testNamespace;
+            this.podLabels = podLabels;
         }
 
         public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
+            var pl = podLabels.GetLabelsForAppName(recipeFactory.AppName);
+
             return K8s(controller =>
             {
                 var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
@@ -30,7 +34,12 @@ namespace KubernetesWorkflow
                 var runningPod = controller.BringOnline(recipes, location);
 
                 return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig));
-            });
+            }, pl);
+        }
+
+        public CrashWatcher CreateCrashWatcher(RunningContainer container)
+        {
+            return K8s(controller => controller.CreateCrashWatcher(container));
         }
 
         public void Stop(RunningContainers runningContainers)
@@ -41,11 +50,11 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler)
+        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler);
+                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler, tailLines);
             });
         }
 
@@ -147,14 +156,22 @@ namespace KubernetesWorkflow
 
         private void K8s(Action<K8sController> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels);
             action(controller);
             controller.Dispose();
         }
 
         private T K8s<T>(Func<K8sController, T> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels);
+            var result = action(controller);
+            controller.Dispose();
+            return result;
+        }
+
+        private T K8s<T>(Func<K8sController, T> action, PodLabels labels)
+        {
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, labels);
             var result = action(controller);
             controller.Dispose();
             return result;

KubernetesWorkflow/WorkflowCreator.cs

@@ -10,18 +10,15 @@ namespace KubernetesWorkflow
         private readonly KnownK8sPods knownPods = new KnownK8sPods();
         private readonly K8sCluster cluster;
         private readonly BaseLog log;
+        private readonly PodLabels podLabels;
         private readonly string testNamespace;
 
-        public WorkflowCreator(BaseLog log, Configuration configuration)
-            : this(log, configuration, Guid.NewGuid().ToString().ToLowerInvariant())
-        {
-        }
-
-        public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespacePostfix)
+        public WorkflowCreator(BaseLog log, Configuration configuration, PodLabels podLabels, string testNamespace)
         {
             cluster = new K8sCluster(configuration);
             this.log = log;
-            testNamespace = testNamespacePostfix;
+            this.podLabels = podLabels;
+            this.testNamespace = testNamespace.ToLowerInvariant();
         }
 
         public StartupWorkflow CreateWorkflow()
@@ -29,7 +26,7 @@ namespace KubernetesWorkflow
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
                                                                     containerNumberSource);
 
-            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);
+            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace, podLabels);
         }
     }
 }

Logging/ApplicationIds.cs

@@ -0,0 +1,20 @@
+namespace Logging
+{
+    public class ApplicationIds
+    {
+        public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId, string grafanaId)
+        {
+            CodexId = codexId;
+            GethId = gethId;
+            PrometheusId = prometheusId;
+            CodexContractsId = codexContractsId;
+            GrafanaId = grafanaId;
+        }
+
+        public string CodexId { get; }
+        public string GethId { get; }
+        public string PrometheusId { get; }
+        public string CodexContractsId { get; }
+        public string GrafanaId { get; }
+    }
+}

Logging/NameUtils.cs

@@ -22,6 +22,7 @@ namespace Logging
         public static string GetRawFixtureName()
         {
             var test = TestContext.CurrentContext.Test;
+            if (test.ClassName!.Contains("AdhocContext")) return "none";
             var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
             return className.Replace('.', '-');
         }
@@ -29,6 +30,7 @@ namespace Logging
         public static string GetCategoryName()
         {
             var test = TestContext.CurrentContext.Test;
+            if (test.ClassName!.Contains("AdhocContext")) return "none";
             return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.'));
         }
 
@@ -45,7 +47,7 @@ namespace Logging
         private static string GetEnvVar(string name)
         {
             var v = Environment.GetEnvironmentVariable(name);
-            if (string.IsNullOrEmpty(v)) return $"EnvVar'{name}'NotSet";
+            if (string.IsNullOrEmpty(v)) return $"EnvVar-{name}-NotSet";
             return v;
         }
 

Logging/StatusLog.cs

@@ -14,7 +14,7 @@ namespace Logging
             fixtureName = NameUtils.GetRawFixtureName();
         }
 
-        public void ConcludeTest(string resultStatus, string testDuration, string codexId)
+        public void ConcludeTest(string resultStatus, string testDuration, ApplicationIds applicationIds)
         {
             Write(new StatusLogJson
             {
@@ -22,7 +22,11 @@ namespace Logging
                 runid = NameUtils.GetRunId(),
                 status = resultStatus,
                 testid = NameUtils.GetTestId(),
-                codexid = codexId,
+                codexid = applicationIds.CodexId,
+                gethid = applicationIds.GethId,
+                prometheusid = applicationIds.PrometheusId,
+                codexcontractsid = applicationIds.CodexContractsId,
+                grafanaid = applicationIds.GrafanaId,
                 category = NameUtils.GetCategoryName(),
                 fixturename = fixtureName,
                 testname = NameUtils.GetTestMethodName(),
@@ -53,6 +57,10 @@ namespace Logging
         public string status { get; set; } = string.Empty;
         public string testid { get; set; } = string.Empty; 
         public string codexid { get; set; } = string.Empty;
+        public string gethid { get; set; } = string.Empty;
+        public string prometheusid { get; set; } = string.Empty;
+        public string codexcontractsid { get; set; } = string.Empty;
+        public string grafanaid { get; set; } = string.Empty;
         public string category { get; set; } = string.Empty;
         public string fixturename { get; set; } = string.Empty;
         public string testname { get; set; } = string.Empty;

LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs

@@ -1,5 +1,4 @@
-using DistTestCore.Helpers;
-using DistTestCore;
+using DistTestCore;
 using NUnit.Framework;
 
 namespace TestsLong.DownloadConnectivityTests
@@ -16,7 +15,7 @@ namespace TestsLong.DownloadConnectivityTests
         {
             for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
 
-            PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
+            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
         }
     }
 }

README.md

@@ -3,7 +3,6 @@
 
 Using a common dotnet unit-test framework and a few other libraries, this project allows you to write tests that use multiple Codex node instances in various configurations to test the distributed system in a controlled, reproducible environment.
 
-
 Nim-Codex: https://github.com/codex-storage/nim-codex  
 Dotnet: v6.0  
 Kubernetes: v1.25.4  
@@ -23,7 +22,7 @@ Test executing can be configured using the following environment variables.
 |----------------|------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|
 | KUBECONFIG     | Optional path (abs or rel) to kubeconfig YAML file. When null, uses system default (docker-desktop) kubeconfig if available.                         | (null)              |
 | LOGPATH        | Path (abs or rel) where log files will be saved.                                                                                                     | "CodexTestLogs"     |
-| LOGDEBUG       | When "true", enables additional test-runner debug log output.                                                                                                    | "false"             |
+| LOGDEBUG       | When "true", enables additional test-runner debug log output.                                                                                        | "false"             |
 | DATAFILEPATH   | Path (abs or rel) where temporary test data files will be saved.                                                                                     | "TestDataFiles"     |
 | LOGLEVEL       | Codex log-level. (case-insensitive)                                                                                                                  | "Trace"             |
 | RUNNERLOCATION | Use "ExternalToCluster" when test app is running outside of the k8s cluster. Use "InternalToCluster" when tests are run from inside a pod/container. | "ExternalToCluster" |
@@ -32,10 +31,10 @@ Test executing can be configured using the following environment variables.
 Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs.
 
 ## How to contribute tests
-An important goal of the test infra is to provide a simple, accessible way for developers to write their tests. If you want to contribute tests for Codex, please follow the steps [HERE](/CONTRIBUTINGTESTS.MD).
+An important goal of the test infra is to provide a simple, accessible way for developers to write their tests. If you want to contribute tests for Codex, please follow the steps [HERE](/CONTRIBUTINGTESTS.md).
 
 ## Run the tests on your machine
-Creating tests is much easier when you can debug them on your local system. This is possible, but requires some set-up. If you want to be able to run the tests on your local system, follow the steps [HERE](/LOCALSETUP.MD). Please note that tests which require explicit node locations cannot be executed locally. (Well, you could comment out the location statements and then it would probably work. But that might impact the validity/usefulness of the test.)
+Creating tests is much easier when you can debug them on your local system. This is possible, but requires some set-up. If you want to be able to run the tests on your local system, follow the steps [HERE](/docs/LOCALSETUP.md). Please note that tests which require explicit node locations cannot be executed locally. (Well, you could comment out the location statements and then it would probably work. But that might impact the validity/usefulness of the test.)
 
 ## Missing functionality
 Surely the test-infra doesn't do everything we'll need it to do. If you're running into a limitation and would like to request a new feature for the test-infra, please create an issue.

Tests/BasicTests/ContinuousSubstitute.cs

@@ -0,0 +1,194 @@
+using DistTestCore;
+using Logging;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.BasicTests
+{
+    [TestFixture]
+    public class ContinuousSubstitute : AutoBootstrapDistTest
+    {
+        [Test]
+        [UseLongTimeouts]
+        public void ContinuousTestSubstitute()
+        {
+            var group = SetupCodexNodes(5, o => o
+                    .EnableMetrics()
+                    .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
+                    .WithBlockTTL(TimeSpan.FromMinutes(2))
+                    .WithStorageQuota(3.GB()));
+
+            var nodes = group.Cast<OnlineCodexNode>().ToArray();
+
+            foreach (var node in nodes)
+            {
+                node.Marketplace.MakeStorageAvailable(
+                size: 1.GB(),
+                minPricePerBytePerSecond: 1.TestTokens(),
+                maxCollateral: 1024.TestTokens(),
+                maxDuration: TimeSpan.FromMinutes(5));
+            }
+
+            var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
+            while (DateTime.UtcNow < endTime)
+            {
+                var allNodes = nodes.ToList();
+                var primary = allNodes.PickOneRandom();
+                var secondary = allNodes.PickOneRandom();
+
+                Log("Run Test");
+                PerformTest(primary, secondary);
+
+                Thread.Sleep(TimeSpan.FromSeconds(5));
+            }
+        }
+
+        private ByteSize fileSize = 80.MB();
+
+        private void PerformTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
+        {
+            ScopedTestFiles(() =>
+            {
+                var testFile = GenerateTestFile(fileSize);
+
+                var contentId = primary.UploadFile(testFile);
+
+                var downloadedFile = secondary.DownloadContent(contentId);
+
+                testFile.AssertIsEqual(downloadedFile);
+            });
+        }
+        
+        [Test]
+        public void HoldMyBeerTest()
+        {
+            var blockExpirationTime = TimeSpan.FromMinutes(3);
+            var group = SetupCodexNodes(3, o => o
+                    .EnableMetrics()
+                    .WithBlockTTL(blockExpirationTime)
+                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
+                    .WithBlockMaintenanceNumber(10000)
+                    .WithStorageQuota(2000.MB()));
+
+            var nodes = group.Cast<OnlineCodexNode>().ToArray();
+
+            var endTime = DateTime.UtcNow + TimeSpan.FromHours(24);
+
+            var filesize = 80.MB();
+            double codexDefaultBlockSize = 31 * 64 * 33;
+            var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
+            var sizeInBytes = filesize.SizeInBytes;
+            Assert.That(numberOfBlocks, Is.EqualTo(1282));
+
+            var startTime = DateTime.UtcNow;
+            var successfulUploads = 0;
+            var successfulDownloads = 0;
+
+            while (DateTime.UtcNow < endTime)
+            {
+                foreach (var node in nodes)
+                {
+                    try
+                    {
+                        Thread.Sleep(TimeSpan.FromSeconds(5));
+
+                        ScopedTestFiles(() =>
+                        {
+                            var uploadStartTime = DateTime.UtcNow;
+                            var file = GenerateTestFile(filesize);
+                            var cid = node.UploadFile(file);
+
+                            var cidTag = cid.Id.Substring(cid.Id.Length - 6);
+                            Measure("upload-log-asserts", () =>
+                            {
+                                var uploadLog = node.DownloadLog(tailLines: 50000);
+
+                                var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
+                                uploadLog.DeleteFile();
+
+                                var storeLine = GetLineForCidTag(storeLines, cidTag);
+                                AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
+                            });
+                            successfulUploads++;
+
+                            var uploadTimeTaken = DateTime.UtcNow - uploadStartTime;
+                            if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10)))
+                            {
+                                Assert.Fail("Upload took too long. Blocks already expired.");
+                            }
+
+                            var dl = node.DownloadContent(cid);
+                            file.AssertIsEqual(dl);
+
+                            Measure("download-log-asserts", () =>
+                            {
+                                var downloadLog = node.DownloadLog(tailLines: 50000);
+
+                                var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
+                                downloadLog.DeleteFile();
+
+                                var sentLine = GetLineForCidTag(sentLines, cidTag);
+                                AssertSentLineContains(sentLine, sizeInBytes);
+                            });
+                            successfulDownloads++;
+                        });
+                    }
+                    catch
+                    {
+                        var testDuration = DateTime.UtcNow - startTime;
+                        Log("Test failed. Delaying shut-down by 30 seconds to collect metrics.");
+                        Log($"Test failed after {Time.FormatDuration(testDuration)} and {successfulUploads} successful uploads and {successfulDownloads} successful downloads");
+                        Thread.Sleep(TimeSpan.FromSeconds(30));
+                        throw;
+                    }
+                }
+
+                Thread.Sleep(TimeSpan.FromSeconds(5));
+            }
+        }
+
+        private void AssertSentLineContains(string sentLine, long sizeInBytes)
+        {
+            var tag = "bytes=";
+            var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length);
+            var bytes = Convert.ToInt64(token);
+            Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'");
+        }
+
+        private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes)
+        {
+            var tokens = storeLine.Split(" ");
+
+            var blocksToken = GetToken(tokens, "blocks=");
+            var sizeToken = GetToken(tokens, "size=");
+            if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine);
+            if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine);
+
+            var blocks = Convert.ToInt64(blocksToken);
+            var size = Convert.ToInt64(sizeToken?.Replace("'NByte", ""));
+
+            var lineLog = $" Line: '{storeLine}'";
+            Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog);
+            Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog);
+        }
+
+        private string GetLineForCidTag(string[] lines, string cidTag)
+        {
+            var result = lines.SingleOrDefault(l => l.Contains(cidTag));
+            if (result == null)
+            {
+                Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'");
+                throw new Exception();
+            }
+
+            return result;
+        }
+
+        private string? GetToken(string[] tokens, string tag)
+        {
+            var token = tokens.SingleOrDefault(t => t.StartsWith(tag));
+            if (token == null) return null;
+            return token.Substring(tag.Length);
+        }
+    }
+}

Tests/BasicTests/ExampleTests.cs

@@ -1,9 +1,5 @@
 using DistTestCore;
-using DistTestCore.Codex;
-using DistTestCore.Marketplace;
-using Newtonsoft.Json;
 using NUnit.Framework;
-using Utils;
 
 namespace Tests.BasicTests
 {

Tests/BasicTests/TwoClientTests.cs

@@ -8,7 +8,7 @@ namespace Tests.BasicTests
     public class TwoClientTests : DistTest
     {
         [Test]
-        public void TwoClientsOnePodTest()
+        public void TwoClientTest()
         {
             var group = SetupCodexNodes(2);
 
@@ -18,15 +18,6 @@ namespace Tests.BasicTests
             PerformTwoClientTest(primary, secondary);
         }
 
-        [Test]
-        public void TwoClientsTwoPodsTest()
-        {
-            var primary = SetupCodexNode();
-            var secondary = SetupCodexNode();
-
-            PerformTwoClientTest(primary, secondary);
-        }
-
         [Test]
         public void TwoClientsTwoLocationsTest()
         {

Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -6,15 +6,36 @@ namespace Tests.DownloadConnectivityTests
     [TestFixture]
     public class FullyConnectedDownloadTests : AutoBootstrapDistTest
     {
+        [Test]
+        public void MetricsDoesNotInterfereWithPeerDownload()
+        {
+            SetupCodexNodes(2, s => s.EnableMetrics());
+
+            AssertAllNodesConnected();
+        }
+
+        [Test]
+        public void MarketplaceDoesNotInterfereWithPeerDownload()
+        {
+            SetupCodexNodes(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens()));
+
+            AssertAllNodesConnected();
+        }
+
         [Test]
         [Combinatorial]
         public void FullyConnectedDownloadTest(
             [Values(1, 3, 5)] int numberOfNodes,
             [Values(1, 10)] int sizeMBs)
         {
-            for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
+            SetupCodexNodes(numberOfNodes);
 
-            PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
+            AssertAllNodesConnected(sizeMBs);
+        }
+
+        private void AssertAllNodesConnected(int sizeMBs = 10)
+        {
+            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
         }
     }
 }

Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs

@@ -1,5 +1,4 @@
 using DistTestCore;
-using DistTestCore.Helpers;
 using NUnit.Framework;
 
 namespace Tests.PeerDiscoveryTests
@@ -47,7 +46,7 @@ namespace Tests.PeerDiscoveryTests
 
         private void AssertAllNodesConnected()
         {
-            PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
+            CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
         }
     }
 }

Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -1,5 +1,4 @@
 using DistTestCore;
-using DistTestCore.Helpers;
 using NUnit.Framework;
 
 namespace Tests.PeerDiscoveryTests
@@ -17,23 +16,36 @@ namespace Tests.PeerDiscoveryTests
             Assert.That(result.IsPeerFound, Is.False);
         }
 
+        [Test]
+        public void MetricsDoesNotInterfereWithPeerDiscovery()
+        {
+            SetupCodexNodes(2, s => s.EnableMetrics());
+
+            AssertAllNodesConnected();
+        }
+
+        [Test]
+        public void MarketplaceDoesNotInterfereWithPeerDiscovery()
+        {
+            SetupCodexNodes(2, s => s.EnableMarketplace(1000.TestTokens()));
+
+            AssertAllNodesConnected();
+        }
+
         [TestCase(2)]
         [TestCase(3)]
         [TestCase(10)]
         [TestCase(20)]
-        public void VariableNodesInPods(int number)
+        public void VariableNodes(int number)
         {
-            for (var i = 0; i < number; i++)
-            {
-                SetupCodexNode();
-            }
+            SetupCodexNodes(number);
 
             AssertAllNodesConnected();
         }
 
         private void AssertAllNodesConnected()
         {
-            PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
+            CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
         }
     }
 }

Tests/Tests.csproj

@@ -13,6 +13,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\ContinuousTests\ContinuousTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
   </ItemGroup>
 

docker/README.md

@@ -1,138 +0,0 @@
-# Run tests with Docker in Kubernetes
-
- We may [run tests on local](../LOCALSETUP.MD) or remote Kubernetes cluster. Local cluster flow uses direct access to nodes ports and this is why we introduced a different way to check services ports, for more information please see [Tests run modes](../../../issues/20). Configuration option `RUNNERLOCATION` is responsible for that.
-
-
- For local run it is easier to install .Net and run tests on Docker Desktop Kubernetes cluster. In case of remote run we do not expose services via Ingress Controller and we can't access cluster nodes, this is why we should run tests only inside the Kubernetes.
-
- We can run tests on remote cluster in the following ways
-
-#### Run pod inside the cluster using generic .Net image
-
-<details>
-<summary>steps</summary>
-
-1. Create dist-tests-runner.yaml
-   ```yaml
-      --
-      apiVersion: v1
-      kind: Pod
-      metadata:
-      name: dist-tests-runner
-      namespace: default
-      spec:
-      containers:
-      - name: dotnet
-         image: mcr.microsoft.com/dotnet/sdk:7.0
-         command: ["sleep", "infinity"]
-   ```
-2. Deploy pod in the cluster
-   ```shell
-   kubectl apply -f dist-tests-runner.yaml
-   ```
-
-3. Copy kubeconfig to the pod
-   ```shell
-   kubectl cp kubeconfig.yaml dist-tests-runner:/opt
-   ```
-
-4. Exec into the pod via kubectl or [OpenLens](https://github.com/MuhammedKalkan/OpenLens)
-   ```shell
-   kubectl exec -it dist-tests-runner -- bash
-   ```
-
-5. Clone repository inside the pod
-   ```shell
-   git clone https://github.com/codex-storage/cs-codex-dist-tests.git
-   ```
-
-6. Update kubeconfig option in config file
-   ```shell
-   cd cs-codex-dist-tests
-   vi DistTestCore/Configuration.cs
-   ```
-   ```dotnet
-   GetNullableEnvVarOrDefault("KUBECONFIG", "/opt/kubeconfig.yaml")
-   ```
-
-7. Run tests
-   ```shell
-   dotnet test Tests
-   ```
-
-8. Check the results and analyze the logs
-</details>
-
-#### Run pod inside the cluster using [prepared Docker images](https://hub.docker.com/r/codexstorage/cs-codex-dist-tests/tags)
-
- Before the run we should create some objects inside the cluster
- 1. Namespace where we will run the image
- 2. [Service Account to run tests inside the cluster](https://github.com/codex-storage/cs-codex-dist-tests/issues/21)
- 3. Secret with kubeconfig for created SA
- 4. Configmap with custom app config if required
-
- For more information please see [Manual run inside Kubernetes via Job](../../../issues/7)
-
- Then we need to create a manifest to run the pod
- <details>
- <summary>runner.yaml</summary>
-
- ```yaml
- ---
- apiVersion: v1
- kind: Pod
- metadata:
-   name: dist-tests-runner
-   namespace: cs-codex-dist-tests
-   labels:
-     name: cs-codex-dist-tests
- spec:
-   containers:
-   - name: cs-codex-dist-tests
-     image: codexstorage/cs-codex-dist-tests:sha-671ee4e
-     env:
-     - name: RUNNERLOCATION
-       value: InternalToCluster
-     - name: KUBECONFIG
-       value: /opt/kubeconfig.yaml
-     - name: CONFIG
-       value: "/opt/Configuration.cs"
-     - name: CONFIG_SHOW
-       value: "true"
-     volumeMounts:
-     - name: kubeconfig
-       mountPath: /opt/kubeconfig.yaml
-       subPath: kubeconfig.yaml
-     - name: config
-       mountPath: /opt/Configuration.cs
-       subPath: Configuration.cs
-     - name: logs
-       mountPath: /var/log/cs-codex-dist-tests
-     # command:
-     # - "dotnet"
-     # - "test"
-     # - "Tests"
-   restartPolicy: Never
-   volumes:
-     - name: kubeconfig
-       secret:
-         secretName: cs-codex-dist-tests-app-kubeconfig
-     - name: config
-       configMap:
-         name: cs-codex-dist-tests
-     - name: logs
-       hostPath:
-         path: /var/log/cs-codex-dist-tests
- ```
- For more information about pod variables please see [job.yaml](job.yaml).
- </details>
-
- And then apply it
- ```shell
- kubectl apply -f runner.yaml
- ```
-
- After the pod run, custom [entrypoint](docker-entrypoint.sh) will do the following
- 1. Clone repository
- 2. Switch to the specific branch - `master` by default
- 3. Run all tests - `dotnet test`

docker/job.yaml

@@ -14,7 +14,8 @@ spec:
     spec:
       containers:
       - name: ${NAMEPREFIX}-runner
-        image: codexstorage/cs-codex-dist-tests:sha-300b91e
+        image: codexstorage/cs-codex-dist-tests:latest
+        imagePullPolicy: Always
         env:
         - name: RUNNERLOCATION
           value: InternalToCluster

docs/AUTOMATION.md

@@ -0,0 +1,253 @@
+# Distributed Tests automation
+
+ 1. [Description](#description)
+ 2. [Architecture](#architecture)
+ 3. [Run on remote cluster](#run-on-remote-cluster)
+    - [Kubernetes cluster pre-configuration](#kubernetes-cluster-pre-configuration)
+    - [Run tests manually](#run-tests-manually)
+    - [Run tests automatically](#run-tests-automatically)
+    - [Tests logs](#tests-logs)
+
+
+## Description
+
+ The goal of [Distributed System Tests for Nim-Codex](../../) is to test how [Codex](https://codex.storage) works in different topologies in the distributed network and to be able to detect regressions during development.
+
+ We can [run Tests locally](LOCALSETUP.md) and it works well, but in order to scale that we may need to run Tests in an automatic way using remote Kubernetes cluster.
+
+ Initially, we are considering to run dist-tests on [nim-codex](https://github.com/codex-storage/nim-codex) master branch merge, to be able to determine regressions. And we also working on [Continuous Tests](/ContinuousTests) which are called to detect issues on continuous Codex runs.
+
+
+## Architecture
+
+<img src="Architecture.png" alt="Architecture" width="800"/>
+
+```
+                                        Logs     --> Kibana
+                                      /                |
+GitHub --> CI --> Kubernetes -->  Job    Prometheus  Elaticsearch
+      \                          /    \  /         \   |
+        ------------------------        Metrics  --> Grafana
+```
+
+
+### Components
+
+| Component                               | Description                             |
+| --------------------------------------- | --------------------------------------- |
+| [cs-codex-dist-tests](/)                | Distributed System Tests                |
+| [Kubernetes](https://kubernetes.io)     | Environment where we run Tests          |
+| [Vector](https://vector.dev)            | Ship logs to the Elasticsearch          |
+| [Elasticsearch](https://www.elastic.co) | Store and index logs                    |
+| [Kibana](https://www.elastic.co)        | Discover the logs in Elasticsearch      |
+| [Grafana](https://grafana.com)          | Visualize tests run results and metrics |
+| [Prometheus](https://prometheus.io)     | Collect and store metrics               |
+
+> Note: These components are not mentioned on the diagram and provided to understand what do we have under the hood
+
+
+## Run on remote cluster
+
+ In case of local run we use [Docker Desktop Kubernetes cluster](https://docs.docker.com/desktop/kubernetes/) and during the services checks, app connect directly to the cluster worker nodes and perform ports check. And in case of remote cluster, it would be required to configure services ports exposing using Ingress Controller or run tests directly inside the cluster.
+
+ Now, we have a configuration key `RUNNERLOCATION` which change the logic of the services ports check and when we run tests inside remote cluster we should set it to the `InternalToCluster`.
+
+ As for now, it was decided to run tests inside the remote Kubernetes cluster using CI because
+ - Stable connection from app to the Kubernetes API
+ - Independence from the CI limitations for long runs
+ - Easy, fast, configurable and detachable run
+
+ Because tests are run on remote cluster we need a way to see their execution status and to analyze the logs as well. For that we use Elasticsearch, Kibana and Grafana with logs shipped by Vector. Please see [Tests logs](#tests-logs) for more information.
+
+ Now we can [Run tests manually](#run-tests-manually) and [Run tests automatically](#run-tests-automatically) on [remote Kubernetes cluster which requires to be pre-configured](#kubernetes-cluster-pre-configuration).
+
+
+### Kubernetes cluster pre-configuration
+
+ Before running the tests on remote Kubernetes cluster we performed [manual pre-configuration](../../../issues/7) and it was require to
+ 1. [Create a namespace](../../../issues/7)
+ 2. [Create kubeconfig for App](../../../issues/21)
+ 3. [Create a secret with created kubeconfig](../../../issues/21)
+ 4. [Create kubeconfig for GitHub Actions](../../../issues/19)
+
+
+### Run tests manually
+
+ **To run tests manually we have the following requirements**
+ 1. Get kubeconfig - to access the cluster
+ 2. Install [kubectl](https://kubernetes.io/docs/tasks/tools/) - to create resources in the cluster
+ 3. Install [OpenLens](https://github.com/MuhammedKalkan/OpenLens) - to browse cluster resources
+
+ **And to run the tests we should perform the following steps**
+ 1. Create a Pod in the cluster, in the `default` namespace and consider to use your own value for `metadata.name`
+    <details>
+    <summary>pod.yaml</summary>
+
+    ```yaml
+    ---
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: dist-tests-runner
+      namespace: default
+      labels:
+        app: dist-tests-runner
+        launched: manually
+    spec:
+      containers:
+      - name: dotnet
+        image: mcr.microsoft.com/dotnet/sdk:7.0
+        env:
+        - name: RUNNERLOCATION
+          value: InternalToCluster
+        - name: KUBECONFIG
+          value: /opt/kubeconfig.yaml
+        command: ["sleep", "infinity"]
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /opt/kubeconfig.yaml
+          subPath: kubeconfig.yaml
+        - name: logs
+          mountPath: /var/log/cs-codex-dist-tests
+      restartPolicy: Never
+      volumes:
+        - name: kubeconfig
+          secret:
+            secretName: cs-codex-dist-tests-app-kubeconfig
+        - name: logs
+          hostPath:
+            path: /var/log/cs-codex-dist-tests
+    ```
+    </details>
+
+    ```shell
+    kubectl apply -f pod.yaml
+    ```
+
+ 2. Exec into the Pod using the name you set in the previous step
+    ```shell
+    # kubectl
+    kubectl exec -it dist-tests-runner -- bash
+
+    # OpenLens
+    OpenLens --> Pods --> dist-tests-runner --> "Press on it" --> Pod Shell
+    ```
+
+ 3. Clone dist-tests repository
+    ```shell
+    folder="/opt/dist-tests"
+
+    git clone https://github.com/codex-storage/cs-codex-dist-tests.git $folder
+    cd $folder
+    ```
+
+ 4. Define variables - optional
+    ```shell
+    # RUNNERLOCATION                              # defined at Pod creation
+    # KUBECONFIG                                  # defined at Pod creation
+    export LOGPATH="/var/log/cs-codex-dist-tests" # Logs from that location will be send in Elasticsearch
+    export RUNID=$(date +%Y%m%d-%H%M%S)           # Run ID to show in Kibana/Grafana
+    export TESTID=$(git rev-parse --short HEAD)   # Test ID to show in Kibana/Grafana
+    ```
+
+ 5. Run tests
+    ```shell
+    # All tests
+    dotnet test
+
+    # Short tests
+    dotnet test Tests
+
+    # Long tests
+    dotnet test LongTests
+
+    # Specific test
+    dotnet test --filter=CodexLogExample
+    ```
+
+ 6. We can see in OpenLens Pods started by dist-tests app
+
+ 7. If we set `LOGPATH` to a location specified above, we should be able to see tests execution status in Kibana/Grafana. For more information, please see [Tests logs](#tests-logs).
+
+
+### Run tests automatically
+
+ Now we use GitHub Actions to trigger dist-tests run manually and considering to run them on [nim-codex](https://github.com/codex-storage/nim-codex) master branch merge.
+
+ **It works in the following way**
+ 1. Github Actions secrets contains [kubeconfig to interact with the Kubernetes cluster](../../../issues/19).
+ 2. GitHub Actions [workflow](../../../actions/workflows/dist-tests.yaml) uses kubectl to create Kubernetes Job based on the [job.yaml](/docker/job.yaml) manifest file. It also accept optional inputs at run
+    - `source`     - Dist-tests source repository
+    - `branch`     - Repository branch
+    - `namespace`  - Namespace where Dist-test runner will de created
+    - `nameprefix` - Dist-test runner name prefix
+
+ 3. Kubernetes Job will run the Pod with a custom [Docker image](/docker/Dockerfile) - [codexstorage/cs-codex-dist-tests](https://hub.docker.com/r/codexstorage/cs-codex-dist-tests/tags).  
+  Image [entrypoint](/docker/docker-entrypoint.sh) is customizable and we can pass the following variables
+    - `SOURCE` - Dist-tests source repository, useful when we work with forks - `default="current repository"`
+    - `BRANCH` - Repository branch, useful when we would like to run tests from the custom branch - `default="master"`
+    - `FOLDER` - Where to clone repository and it is done just to organize the things - `default="/opt/dist-tests"`
+
+    > **Note:** Variables `SOURCE` and `BRANCH` passed by GitHub Actions to the Kubernetes Job and then to the Pod.
+
+ 4. Job manifest is setting all required variables which are part of the Pod run
+    ```shell
+    RUNNERLOCATION=InternalToCluster
+    KUBECONFIG=/opt/kubeconfig.yaml
+    LOGPATH=/var/log/cs-codex-dist-tests
+    NAMESPACE=default
+    SOURCE=current repository
+    BRANCH=master
+    RUNID=datetime
+    TESTID=short sha
+    ```
+
+ 5. Dist-tests runner will use [kubeconfig to interact with the Kubernetes and run the tests](../../../issues/21), which is set by `KUBECONFIG` variable.
+
+ 6. Runner will execute all tests and will write the logs to the `/var/log/cs-codex-dist-tests` folder.
+
+ 7. Kubernetes Job status will be changed from `Running` to the `Completed` or `Failed`.
+
+ 8. Vector will ship the logs to the Elasticsearch.
+
+ 9. We can check execution status in Kibana and Grafana. For more information, please see [Tests logs](#tests-logs).
+
+
+### Tests logs
+
+ > **Note:** This part is not finished yet and it is under development
+
+ We use Elasticsearch to store and discover the logs of the tests execution and Codex Pods which are run during the tests.
+
+ We can use [Kibana](#kibana) to discover all the logs and [Grafana](#grafana) to see tests execution status.
+
+
+#### Endpoints
+
+ | App     | URL                                                                          | Authentication |
+ | ------- | ---------------------------------------------------------------------------- | -------------- |
+ | Kibana  | [kibana.dist-tests.codex.storage](https://kibana.dist-tests.codex.storage)   | GitHub account |
+ | Grafana | [grafana.dist-tests.codex.storage](https://grafana.dist-tests.codex.storage) | GitHub account |
+
+
+#### Kibana
+
+ As for now, we have the following indices in Kibana
+ | # | Index Pattern              | Description                         |
+ | - | -------------------------- | ----------------------------------- |
+ | 1 | `dist-tests-status*`       | Dist-tests execution status logs    |
+ | 2 | `dist-tests-logs*`         | Dist-tests logs collected by runner |
+ | 3 | `dist-tests-pods*`         | Dist-tests Pods logs                |
+ | 4 | `continuous-tests-status*` | Dist-tests execution status logs    |
+ | 5 | `continuous-tests-logs*`   | Dist-tests logs collected by runner |
+ | 6 | `continuous-tests-pods*`   | Dist-tests Pods logs                |
+ | 7 | `kubernetes*`              | All Kubernetes Pods logs            |
+
+
+#### Grafana
+
+ As for now, we have the following Dashboards in Grafana
+ | # | Dashboard                  |
+ | - | -------------------------- |
+ | 1 | Distributed Tests - Status |
+ | 2 | Continuous Tests - Status  |