Uncommented tests

.github/workflows/dist-tests.yaml

@@ -15,19 +15,19 @@ on:
   workflow_dispatch:
     inputs:
       branch:
-        description: Branch (master)
+        description: Branch
         required: false
         type: string
       source:
-        description: Repository with tests (current)
+        description: Repository with tests
         required: false
         type: string
       nameprefix:
-        description: Runner prefix (cs-codex-dist-tests)
+        description: Runner job/pod name prefix
         required: false
         type: string
       namespace:
-        description: Runner namespace (cs-codex-dist-tests)
+        description: Kubernetes namespace for runner
         required: false
         type: string
 
@@ -56,8 +56,6 @@ jobs:
           [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
           [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
           [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
-          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
-          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
 
       - name: Kubectl - Install ${{ env.KUBE_VERSION }}
         uses: azure/setup-kubectl@v3
@@ -71,4 +69,5 @@ jobs:
 
       - name: Kubectl - Create Job
         run: |
+          export RUNID=$(date +%Y%m%d-%H%M%S)
           envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -

.github/workflows/docker-reusable.yml

@@ -1,177 +0,0 @@
-name: Docker - Reusable
-
-
-on:
-  workflow_call:
-    inputs:
-      docker_file:
-        default: docker/Dockerfile
-        description: Dockerfile
-        required: false
-        type: string
-      docker_repo:
-        default: codexstorage/cs-codex-dist-tests
-        description: DockerHub repository
-        required: false
-        type: string
-      tag_latest:
-        default: true
-        description: Set latest tag for Docker images
-        required: false
-        type: boolean
-      tag_sha:
-        default: true
-        description: Set Git short commit as Docker tag
-        required: false
-        type: boolean
-      tag_suffix:
-        default: ''
-        description: Suffix for Docker images tag
-        required: false
-        type: string
-
-
-env:
-  DOCKER_FILE: ${{ inputs.docker_file }}
-  DOCKER_REPO: ${{ inputs.docker_repo }}
-  TAG_LATEST: ${{ inputs.tag_latest }}
-  TAG_SHA: ${{ inputs.tag_sha }}
-  TAG_SUFFIX: ${{ inputs.tag_suffix }}
-
-
-jobs:
-  # Build platform specific image
-  build:
-    strategy:
-      fail-fast: true
-      matrix:
-        target:
-          - os: linux
-            arch: amd64
-          - os: linux
-            arch: arm64
-        include:
-          - target:
-              os: linux
-              arch: amd64
-            builder: ubuntu-22.04
-          - target:
-              os: linux
-              arch: arm64
-            builder: buildjet-4vcpu-ubuntu-2204-arm
-
-    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
-    runs-on: ${{ matrix.builder }}
-    env:
-      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.DOCKER_REPO }}
-
-      - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Build and Push by digest
-        id: build
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ${{ env.DOCKER_FILE }}
-          platforms: ${{ env.PLATFORM }}
-          push: true
-          labels: ${{ steps.meta.outputs.labels }}
-          outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
-
-      - name: Docker - Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-      - name: Docker - Upload digest
-        uses: actions/upload-artifact@v3
-        with:
-          name: digests
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
-
-
-  # Publish multi-platform image
-  publish:
-    name: Publish multi-platform image
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Docker - Variables
-        run: |
-          # Adjust custom suffix when set and
-          if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
-            echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
-          fi
-          # Disable SHA tags on tagged release
-          if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
-            echo "TAG_SHA=false" >>$GITHUB_ENV
-          fi
-          # Handle latest and latest-custom using raw
-          if [[ ${{ env.TAG_SHA }} == "false" ]]; then
-            echo "TAG_LATEST=false" >>$GITHUB_ENV
-            echo "TAG_RAW=true" >>$GITHUB_ENV
-            if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
-              echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
-            else
-              echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
-            fi
-          else
-            echo "TAG_RAW=false" >>$GITHUB_ENV
-          fi
-
-      - name: Docker - Download digests
-        uses: actions/download-artifact@v3
-        with:
-          name: digests
-          path: /tmp/digests
-
-      - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.DOCKER_REPO }}
-          flavor: |
-            latest=${{ env.TAG_LATEST }}
-            suffix=${{ env.TAG_SUFFIX }},onlatest=true
-          tags: |
-            type=semver,pattern={{version}}
-            type=raw,enable=${{ env.TAG_RAW }},value=latest
-            type=sha,enable=${{ env.TAG_SHA }}
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Create manifest list and push
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
-
-      - name: Docker - Inspect image
-        run: |
-          docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}

.github/workflows/docker.yml

@@ -11,12 +11,113 @@ on:
       - docker/Dockerfile
       - docker/docker-entrypoint.sh
       - .github/workflows/docker.yml
-      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
 
+env:
+  DOCKER_FILE: docker/Dockerfile
+  DOCKER_REPO: codexstorage/cs-codex-dist-tests
+
+
 jobs:
-  build-and-push:
-    name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
-    secrets: inherit
+  # Build platform specific image
+  build:
+    strategy:
+      fail-fast: true
+      matrix:
+        target:
+          - os: linux
+            arch: amd64
+          - os: linux
+            arch: arm64
+        include:
+          - target:
+              os: linux
+              arch: amd64
+            builder: ubuntu-22.04
+          - target:
+              os: linux
+              arch: arm64
+            builder: buildjet-4vcpu-ubuntu-2204-arm
+
+    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
+    runs-on: ${{ matrix.builder }}
+    outputs:
+      tags-linux-amd64: ${{ steps.tags.outputs.tags-linux-amd64 }}
+      tags-linux-arm64: ${{ steps.tags.outputs.tags-linux-arm64 }}
+    env:
+      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
+      SUFFIX: ${{ format('{0}-{1}', 'linux', matrix.target.arch) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_REPO }}
+          flavor: |
+            latest=false
+          tags: |
+            type=semver,pattern={{version}},suffix=-${{ env.SUFFIX }}
+            type=sha,suffix=-${{ env.SUFFIX }},enable=${{ !startsWith(github.ref, 'refs/tags/') }}
+
+      - name: Docker - Set tags output
+        id: tags
+        run: |
+          if [[ '${{ matrix.target.os }}' == 'linux' && '${{ matrix.target.arch }}' == 'amd64' ]]; then
+            echo "tags-linux-amd64=${{ steps.meta.outputs.tags }}" >> "$GITHUB_OUTPUT"
+          elif [[ '${{ matrix.target.os }}' == 'linux' && '${{ matrix.target.arch }}' == 'arm64' ]]; then
+            echo "tags-linux-arm64=${{ steps.meta.outputs.tags }}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Build and Push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ${{ env.DOCKER_FILE }}
+          platforms: ${{ env.PLATFORM }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  # Publish single image
+  publish:
+    name: Push single image
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_REPO }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=sha,enable=${{ !startsWith(github.ref, 'refs/tags/') }}
+
+      - name: Docker - Set tags
+        run: |
+          # Transform multi-line tags in to the comma-seperated
+          TAGS=$(echo "${{ steps.meta.outputs.tags }}" | tr '\n' ',' | awk '{gsub(/,$/,"");}1')
+          echo "TAGS=${TAGS}" >>$GITHUB_ENV
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Create and push manifest images
+        uses: Noelware/docker-manifest-action@master
+        with:
+          inputs: ${{ env.TAGS }}
+          images: ${{ needs.build.outputs.tags-linux-amd64 }},${{ needs.build.outputs.tags-linux-arm64 }}
+          push: true

CodexNetDeployer/CodexNodeStarter.cs

@@ -8,23 +8,25 @@ namespace CodexNetDeployer
     public class CodexNodeStarter
     {
         private readonly Configuration config;
+        private readonly WorkflowCreator workflowCreator;
         private readonly TestLifecycle lifecycle;
         private readonly GethStartResult gethResult;
         private string bootstrapSpr = "";
         private int validatorsLeft;
 
-        public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
+        public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
         {
             this.config = config;
+            this.workflowCreator = workflowCreator;
             this.lifecycle = lifecycle;
             this.gethResult = gethResult;
             validatorsLeft = numberOfValidators;
         }
 
-        public CodexNodeStartResult? Start(int i)
+        public RunningContainer? Start(int i)
         {
             Console.Write($" - {i} = ");
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var workflow = workflowCreator.CreateWorkflow();
             var workflowStartup = new StartupConfig();
             workflowStartup.Add(gethResult);
             workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));
@@ -60,7 +62,7 @@ namespace CodexNetDeployer
 
                         if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
                         validatorsLeft--;
-                        return new CodexNodeStartResult(workflow, container, codexAccess);
+                        return container;
                     }
                 }
             }
@@ -84,36 +86,14 @@ namespace CodexNetDeployer
             var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
             marketplaceConfig.AccountIndexOverride = i;
             codexStart.MarketplaceConfig = marketplaceConfig;
-            codexStart.MetricsMode = config.Metrics;
+            codexStart.MetricsEnabled = config.RecordMetrics;
 
             if (config.BlockTTL != Configuration.SecondsIn1Day)
             {
                 codexStart.BlockTTL = config.BlockTTL;
             }
-            if (config.BlockMI != Configuration.TenMinutes)
-            {
-                codexStart.BlockMaintenanceInterval = TimeSpan.FromSeconds(config.BlockMI);
-            }
-            if (config.BlockMN != 1000)
-            {
-                codexStart.BlockMaintenanceNumber = config.BlockMN;
-            }
 
             return codexStart;
         }
     }
-
-    public class CodexNodeStartResult
-    {
-        public CodexNodeStartResult(StartupWorkflow workflow, RunningContainer container, CodexAccess access)
-        {
-            Workflow = workflow;
-            Container = container;
-            Access = access;
-        }
-
-        public StartupWorkflow Workflow { get; }
-        public RunningContainer Container { get; }
-        public CodexAccess Access { get; }
-    }
 }

CodexNetDeployer/Configuration.cs

@@ -1,13 +1,12 @@
 using ArgsUniform;
+using DistTestCore;
 using DistTestCore.Codex;
-using DistTestCore.Metrics;
 
 namespace CodexNetDeployer
 {
     public class Configuration
     {
         public const int SecondsIn1Day = 24 * 60 * 60;
-        public const int TenMinutes = 10 * 60;
 
         [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
         public string KubeConfigFile { get; set; } = "null";
@@ -45,22 +44,11 @@ namespace CodexNetDeployer
         [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
         public int BlockTTL { get; set; } = SecondsIn1Day;
 
-        [Uniform("block-mi", "bmi", "BLOCKMI", false, "Block maintenance interval in seconds. Default is 10 minutes.")]
-        public int BlockMI { get; set; } = TenMinutes;
-
-        [Uniform("block-mn", "bmn", "BLOCKMN", false, "Number of blocks maintained per interval. Default is 1000 blocks.")]
-        public int BlockMN { get; set; } = 1000;
-
-        [Uniform("metrics", "m", "METRICS", false, "[None*, Record, Dashboard]. Determines if metrics will be recorded and if a dashboard service will be created.")]
-        public MetricsMode Metrics { get; set; } = MetricsMode.None;
-
-        [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
-            "set this option to override the label value.")]
-        public string TestsTypePodLabel { get; set; } = "continuous-tests";
-
-        [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment.")]
-        public bool CheckPeerConnection { get; set; } = false;
+        [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
+        public bool RecordMetrics { get; set; } = false;
        
+        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
+
         public List<string> Validate()
         {
             var errors = new List<string>();

CodexNetDeployer/Deployer.cs

@@ -8,29 +8,29 @@ namespace CodexNetDeployer
     public class Deployer
     {
         private readonly Configuration config;
+        private readonly NullLog log;
         private readonly DefaultTimeSet timeset;
-        private readonly PeerConnectivityChecker peerConnectivityChecker;
 
         public Deployer(Configuration config)
         {
             this.config = config;
+            log = new NullLog();
             timeset = new DefaultTimeSet();
-            peerConnectivityChecker = new PeerConnectivityChecker();
         }
 
         public CodexDeployment Deploy()
         {
             Log("Initializing...");
-            var lifecycle = CreateTestLifecycle();
+            var (workflowCreator, lifecycle) = CreateFacilities();
 
             Log("Preparing configuration...");
             // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
             var setup = new CodexSetup(999, config.CodexLogLevel);
             setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
-            setup.MetricsMode = config.Metrics;
+            setup.MetricsEnabled = config.RecordMetrics;
 
             Log("Creating Geth instance and deploying contracts...");
-            var gethStarter = new GethStarter(lifecycle);
+            var gethStarter = new GethStarter(lifecycle, workflowCreator);
             var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
 
             Log("Geth started. Codex contracts deployed.");
@@ -44,23 +44,20 @@ namespace CodexNetDeployer
             Log("Starting Codex nodes...");
 
             // Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
-            var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
-            var startResults = new List<CodexNodeStartResult>();
+            var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, gethResults, config.NumberOfValidators!.Value);
+            var codexContainers = new List<RunningContainer>();
             for (var i = 0; i < config.NumberOfCodexNodes; i++)
             {
-                var result = codexStarter.Start(i);
-                if (result != null) startResults.Add(result);
+                var container = codexStarter.Start(i);
+                if (container != null) codexContainers.Add(container);
             }
 
-            var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container));
+            var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers);
 
-            CheckPeerConnectivity(startResults);
-            CheckContainerRestarts(startResults);
-
-            return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
+            return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
         }
 
-        private TestLifecycle CreateTestLifecycle()
+        private (WorkflowCreator, TestLifecycle) CreateFacilities()
         {
             var kubeConfig = GetKubeConfig(config.KubeConfigFile);
 
@@ -71,25 +68,28 @@ namespace CodexNetDeployer
                 logDebug: false,
                 dataFilesPath: "notUsed",
                 codexLogLevel: config.CodexLogLevel,
-                k8sNamespacePrefix: config.KubeNamespace
+                runnerLocation: config.RunnerLocation
             );
 
-            return new TestLifecycle(new NullLog(), lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty);
+            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
+                k8sNamespacePrefix: config.KubeNamespace,
+                kubeConfigFile: kubeConfig,
+                operationTimeout: timeset.K8sOperationTimeout(),
+                retryDelay: timeset.WaitForK8sServiceDelay());
+
+            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
+            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
+
+            return (workflowCreator, lifecycle);
         }
 
-        private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable<RunningContainer> codexContainers)
+        private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
         {
-            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
+            if (!setup.MetricsEnabled) return null;
 
             Log("Starting metrics service...");
-            var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
-            var prometheusContainer = lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
-
-            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null);
-
-            Log("Starting dashboard service...");
-            var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer, setup);
-            return (prometheusContainer, grafanaStartInfo);
+            var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray());
+            return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
         }
 
         private string? GetKubeConfig(string kubeConfigFile)
@@ -98,35 +98,6 @@ namespace CodexNetDeployer
             return kubeConfigFile;
         }
 
-        private void CheckPeerConnectivity(List<CodexNodeStartResult> codexContainers)
-        {
-            if (!config.CheckPeerConnection) return;
-
-            Log("Starting peer-connectivity check for deployed nodes...");
-            peerConnectivityChecker.CheckConnectivity(codexContainers);
-            Log("Check passed.");
-        }
-
-        private void CheckContainerRestarts(List<CodexNodeStartResult> startResults)
-        {
-            var crashes = new List<RunningContainer>();
-            foreach (var startResult in startResults)
-            {
-                var watcher = startResult.Workflow.CreateCrashWatcher(startResult.Container);
-                if (watcher.HasContainerCrashed()) crashes.Add(startResult.Container);
-            }
-
-            if (!crashes.Any())
-            {
-                Log("Container restart check passed.");
-            }
-            else
-            {
-                Log($"Deployment failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}");
-                throw new Exception("Deployment failed: One or more containers crashed.");
-            }
-        }
-
         private DeploymentMetadata CreateMetadata()
         {
             return new DeploymentMetadata(
@@ -138,10 +109,7 @@ namespace CodexNetDeployer
                 initialTestTokens: config.InitialTestTokens,
                 minPrice: config.MinPrice,
                 maxCollateral: config.MaxCollateral,
-                maxDuration: config.MaxDuration,
-                blockTTL: config.BlockTTL,
-                blockMI: config.BlockMI,
-                blockMN: config.BlockMN);
+                maxDuration: config.MaxDuration);
         }
 
         private void Log(string msg)

CodexNetDeployer/PeerConnectivityChecker.cs

@@ -1,34 +0,0 @@
-using DistTestCore.Helpers;
-using Logging;
-
-namespace CodexNetDeployer
-{
-    public class PeerConnectivityChecker
-    {
-        public void CheckConnectivity(List<CodexNodeStartResult> startResults)
-        {
-            var log = new ConsoleLog();
-            var checker = new PeerConnectionTestHelpers(log);
-            var access = startResults.Select(r => r.Access);
-
-            checker.AssertFullyConnected(access);
-        }
-    }
-
-    public class ConsoleLog : BaseLog
-    {
-        public ConsoleLog() : base(false)
-        {
-        }
-
-        protected override string GetFullName()
-        {
-            return "CONSOLE";
-        }
-
-        public override void Log(string message)
-        {
-            Console.WriteLine(message);
-        }
-    }
-}

CodexNetDeployer/Program.cs

@@ -1,5 +1,6 @@
 using ArgsUniform;
 using CodexNetDeployer;
+using DistTestCore;
 using DistTestCore.Codex;
 using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
@@ -16,6 +17,11 @@ public class Program
         var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
         var config = uniformArgs.Parse(true);
         
+        if (args.Any(a => a == "--external"))
+        {
+            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
+        }
+
         var errors = config.Validate();
         if (errors.Any())
         {
@@ -27,11 +33,10 @@ public class Program
         }
 
         Console.WriteLine("Using images:" + nl +
-            $"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl +
-            $"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl +
-            $"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl +
-            $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl +
-            $"\tGrafana image: '{new GrafanaContainerRecipe().Image}'" + nl);
+            $"\tCodex image: '{CodexContainerRecipe.DockerImage}'" + nl +
+            $"\tCodex Contracts image: '{CodexContractsContainerRecipe.DockerImage}'" + nl +
+            $"\tPrometheus image: '{PrometheusContainerRecipe.DockerImage}'" + nl +
+            $"\tGeth image: '{GethContainerRecipe.DockerImage}'" + nl);
 
         if (!args.Any(a => a == "-y"))
         {
@@ -56,5 +61,8 @@ public class Program
         Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
             "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
             "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
+
+        Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
+            "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
     }
 }

CodexNetDeployer/deploy-continuous-testnet.sh

@@ -3,14 +3,10 @@ dotnet run \
     --kube-namespace=codex-continuous-tests \
     --nodes=5 \
     --validators=3 \
-    --log-level=Trace \
     --storage-quota=2048 \
     --storage-sell=1024 \
     --min-price=1024 \
     --max-collateral=1024 \
     --max-duration=3600000 \
-    --block-ttl=180 \
-    --block-mi=120 \
-    --block-mn=10000 \
-    --metrics=Dashboard \
-    --check-connect=1
+    --block-ttl=120
+

CodexNetDownloader/Configuration.cs

@@ -1,4 +1,5 @@
 using ArgsUniform;
+using DistTestCore;
 using DistTestCore.Codex;
 
 namespace CodexNetDownloader
@@ -15,5 +16,7 @@ namespace CodexNetDownloader
         public string KubeConfigFile { get; set; } = "null";
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
+
+        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
     }
 }

CodexNetDownloader/Program.cs

@@ -15,12 +15,17 @@ public class Program
         var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(PrintHelp, args);
         var config = uniformArgs.Parse(true);
 
+        if (args.Any(a => a == "--external"))
+        {
+            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
+        }
+
         config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
 
         if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
 
         var k8sFactory = new K8sFactory();
-        var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
+        var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
 
         foreach (var container in config.CodexDeployment.CodexContainers)
         {

ContinuousTests/CodexAccessFactory.cs

@@ -12,7 +12,7 @@ namespace ContinuousTests
             return containers.Select(container =>
             {
                 var address = container.ClusterExternalAddress;
-                if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
+                if (config.RunnerLocation == TestRunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
                 return new CodexAccess(log, container, timeSet, address);
             }).ToArray();
         }

ContinuousTests/Configuration.cs

@@ -25,12 +25,9 @@ namespace ContinuousTests
         [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
         public bool StopOnFailure { get; set; } = false;
 
-        [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
-        public bool DownloadContainerLogs { get; set; } = false;
-
         public CodexDeployment CodexDeployment { get; set; } = null!;
 
-        public RunnerLocation RunnerLocation { get; set; }
+        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
     }
 
     public class ConfigLoader
@@ -42,7 +39,10 @@ namespace ContinuousTests
             var result = uniformArgs.Parse(true);
             
             result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
-            result.RunnerLocation = RunnerLocationUtils.DetermineRunnerLocation(result.CodexDeployment.CodexContainers.First());
+            if (args.Any(a => a == "--external"))
+            {
+                result.RunnerLocation = TestRunnerLocation.ExternalToCluster;
+            }
 
             return result;
         }
@@ -57,10 +57,9 @@ namespace ContinuousTests
         private static void PrintHelp()
         {
             var nl = Environment.NewLine;
-            Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
-                "The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
+            Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
 
-            Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
+            Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
                 "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
         }
     }

ContinuousTests/ContinuousTest.cs

@@ -1,7 +1,5 @@
 using DistTestCore;
 using DistTestCore.Codex;
-using DistTestCore.Logs;
-using KubernetesWorkflow;
 using Logging;
 
 namespace ContinuousTests
@@ -89,12 +87,6 @@ namespace ContinuousTests
             return file;
         }
 
-        public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null)
-        {
-            var nodeRunner = new NodeRunner(Nodes, Configuration, TimeSet, Log, Configuration.CodexDeployment.Metadata.KubeNamespace, EthereumAccountIndex);
-            return nodeRunner.DownloadLog(container, tailLines);
-        }
-
         private void DownloadToFile(CodexAccess node, string contentId, TestFile file)
         {
             using var fileStream = File.OpenWrite(file.Filename);

ContinuousTests/ContinuousTestRunner.cs

@@ -24,7 +24,7 @@ namespace ContinuousTests
             startupChecker.Check();
 
             var taskFactory = new TaskFactory();
-            var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "Overview");
+            var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
             overviewLog.Log("Continuous tests starting...");
             var allTests = testFactory.CreateTests();
 
@@ -58,8 +58,8 @@ namespace ContinuousTests
             if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
 
             log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
-            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log);
-            lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources();
+            var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
+            workflowCreator.CreateWorkflow().DeleteTestResources();
         }
     }
 }

ContinuousTests/K8sFactory.cs

@@ -1,12 +1,13 @@
 using DistTestCore.Codex;
 using DistTestCore;
+using KubernetesWorkflow;
 using Logging;
 
 namespace ContinuousTests
 {
     public class K8sFactory
     {
-        public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log)
+        public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation)
         {
             var kubeConfig = GetKubeConfig(kubeConfigFile);
             var lifecycleConfig = new DistTestCore.Configuration
@@ -16,10 +17,19 @@ namespace ContinuousTests
                 logDebug: false,
                 dataFilesPath: dataFilePath,
                 codexLogLevel: CodexLogLevel.Debug,
-                k8sNamespacePrefix: customNamespace
+                runnerLocation: runnerLocation
             );
 
-            return new TestLifecycle(log, lifecycleConfig, timeSet, "continuous-tests", string.Empty);
+            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
+                k8sNamespacePrefix: customNamespace,
+                kubeConfigFile: kubeConfig,
+                operationTimeout: timeSet.K8sOperationTimeout(),
+            retryDelay: timeSet.WaitForK8sServiceDelay());
+
+            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
+            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator);
+
+            return (workflowCreator, lifecycle);
         }
 
         private static string? GetKubeConfig(string kubeConfigFile)

ContinuousTests/NodeRunner.cs

@@ -5,7 +5,6 @@ using KubernetesWorkflow;
 using NUnit.Framework;
 using Logging;
 using Utils;
-using DistTestCore.Logs;
 
 namespace ContinuousTests
 {
@@ -39,25 +38,10 @@ namespace ContinuousTests
             RunNode(bootstrapNode, operation, 0.TestTokens());
         }
 
-        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
-        {
-            var subFile = log.CreateSubfile();
-            var description = container.Name;
-            var handler = new LogDownloadHandler(container, description, subFile);
-
-            log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-
-            var lifecycle = CreateTestLifecycle();
-            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
-            flow.DownloadContainerLog(container, handler, tailLines);
-
-            return new DownloadedLog(subFile, description);
-        }
-
         public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
         {
-            var lifecycle = CreateTestLifecycle();
-            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var (workflowCreator, lifecycle) = CreateFacilities();
+            var flow = workflowCreator.CreateWorkflow();
 
             try
             {
@@ -105,9 +89,9 @@ namespace ContinuousTests
             }
         }
 
-        private TestLifecycle CreateTestLifecycle()
+        private (WorkflowCreator, TestLifecycle) CreateFacilities()
         {
-            return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log);
+            return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation);
         }
     }
 }

ContinuousTests/SingleTestRun.cs

@@ -32,7 +32,7 @@ namespace ContinuousTests
             this.handle = handle;
             this.cancelToken = cancelToken;
             testName = handle.Test.GetType().Name;
-            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
+            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), testName);
 
             nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
             dataFolder = config.DataPath + "-" + Guid.NewGuid();
@@ -138,7 +138,7 @@ namespace ContinuousTests
         private void DownloadClusterLogs()
         {
             var k8sFactory = new K8sFactory();
-            var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
+            var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
 
             foreach (var container in config.CodexDeployment.CodexContainers)
             {
@@ -221,7 +221,7 @@ namespace ContinuousTests
         private DistTestCore.Configuration CreateFileManagerConfiguration()
         {
             return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
-                CodexLogLevel.Error, string.Empty);
+                CodexLogLevel.Error, config.RunnerLocation);
         }
     }
 }

ContinuousTests/StartupChecker.cs

@@ -19,7 +19,7 @@ namespace ContinuousTests
 
         public void Check()
         {
-            var log = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "StartupChecks");
+            var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
             log.Log("Starting continuous test run...");
             log.Log("Checking configuration...");
             PreflightCheck(config);

ContinuousTests/Tests/HoldMyBeerTest.cs

@@ -1,39 +0,0 @@
-using DistTestCore;
-using NUnit.Framework;
-
-namespace ContinuousTests.Tests
-{
-    public class HoldMyBeerTest : ContinuousTest
-    {
-        public override int RequiredNumberOfNodes => 1;
-        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(5);
-        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
-
-        private ContentId? cid;
-        private TestFile file = null!;
-
-        [TestMoment(t: Zero)]
-        public void UploadTestFile()
-        {
-            var metadata = Configuration.CodexDeployment.Metadata;
-            var maxQuotaUseMb = metadata.StorageQuotaMB / 2;
-            var safeTTL = Math.Max(metadata.BlockTTL, metadata.BlockMI) + 30;
-            var runsPerTtl = Convert.ToInt32(safeTTL / RunTestEvery.TotalSeconds);
-            var filesizePerUploadMb = Math.Min(80, maxQuotaUseMb / runsPerTtl);
-            // This filesize should keep the quota below 50% of the node's max.
-
-            var filesize = filesizePerUploadMb.MB();
-            double codexDefaultBlockSize = 31 * 64 * 33;
-            var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
-            Assert.That(numberOfBlocks, Is.EqualTo(1282));
-
-            file = FileManager.GenerateTestFile(filesize);
-
-            cid = UploadFile(Nodes[0], file);
-            Assert.That(cid, Is.Not.Null);
-
-            var dl = DownloadFile(Nodes[0], cid!);
-            file.AssertIsEqual(dl);
-        }
-    }
-}

ContinuousTests/Tests/MarketplaceTest.cs

@@ -60,5 +60,40 @@
 //                file.AssertIsEqual(result);
 //            });
 //        }
+
+//        private void WaitForContractToStart(CodexAccess codexAccess, string purchaseId)
+//        {
+//            var lastState = "";
+//            var waitStart = DateTime.UtcNow;
+//            var filesizeInMb = fileSize.SizeInBytes / (1024 * 1024);
+//            var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0);
+
+//            Log.Log($"{nameof(WaitForContractToStart)} for {Time.FormatDuration(maxWaitTime)}");
+//            while (lastState != "started")
+//            {
+//                CancelToken.ThrowIfCancellationRequested();
+
+//                var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId);
+//                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
+//                if (purchaseStatus != null && purchaseStatus.state != lastState)
+//                {
+//                    lastState = purchaseStatus.state;
+//                    Log.Log("Purchase status: " + statusJson);
+//                }
+
+//                Thread.Sleep(2000);
+
+//                if (lastState == "errored")
+//                {
+//                    Assert.Fail("Contract start failed: " + statusJson);
+//                }
+
+//                if (DateTime.UtcNow - waitStart > maxWaitTime)
+//                {
+//                    Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: {statusJson}");
+//                }
+//            }
+//            Log.Log("Contract started.");
+//        }
 //    }
 //}

ContinuousTests/Tests/TwoClientTest.cs

@@ -6,7 +6,7 @@ namespace ContinuousTests.Tests
     public class TwoClientTest : ContinuousTest
     {
         public override int RequiredNumberOfNodes => 2;
-        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(1);
+        public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
         public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
 
         private ContentId? cid;

ContinuousTests/reports/CodexTestNetReport-July2023.md

@@ -1,38 +0,0 @@
-# Codex Continuous Test-net Report
-Date: 02-08-2023
-
-Report for: 07-2023
-
-
-## Test-net Status
-- Start of month: Offline - faulted
-- End of month: Offline - faulted
-
-(Faulted: Tests fail with such frequency that the information gathered does not justify the cost of leaving the test-net running.)
-
-## Deployment Configuration
-Continous Test-net is deployed to the kubernetes cluster with the following configuration:
-
-5x Codex Nodes:
-- Log-level: Trace
-- Storage quota: 2048 MB
-- Storage sell: 1024 MB
-- Min price: 1024
-- Max collateral: 1024
-- Max duration: 3600000 seconds
-- Block-TTL: 120 seconds
-
-3 of these 5 nodes have:
-- Validator: true
-
-Kubernetes namespace: 'codex-continuous-tests'
-
-## Test Overview
-| Changes        | Test                | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | Status      | Results                                                                              |
-|----------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|--------------------------------------------------------------------------------------|
-| New in 07-2023 | Two-client test     | Every 30 seconds, two nodes are chosen at random. A 10 MB file is generated and uploaded to one. 10 seconds later, it is downloaded from the other. File contents are asserted to be equal.                                                                                                                                                                                                                                                                                                                                                                                    | Faulted     | Test reliably fails after 30 to 45 minutes. Both upload and download failures occur. |
-| New in 07-2023 | Transient-node test | Every 1 minute, a new, transient Codex node is started and bootstrapped against a random node of the test net. A 10 MB file is generated and uploaded to the transient node. The file is then downloaded from a second random test net node and file equality is asserted. After that, the transient node is shut down. 30 seconds later, a new transient Codex node is started and bootstrapped against a third (guaranteed different from first and second) node from the test net. The same file is then downloaded from the new transient node. File equality is asserted. | Not running | Test was not run because the previous more rudamentory test has faulted.             |
-
-## Action Points
-- Codex logs (from these long-running containers) are often incomplete when downloaded after a test failure. A reliable way of maintaining these logs is needed. The logs can quickly explode in size, even for test-nets that run only for a few hours.
-- The distributed testing setup can and has been used to reproduce the failure of the Two-client test in a local environment. Investigation is on-going.

ContinuousTests/run-continuous-tests.sh

@@ -1,6 +1,4 @@
 dotnet run \
     --kube-config=/opt/kubeconfig.yaml \
     --codex-deployment=codex-deployment.json \
-    --keep=1 \
-    --stop=1 \
-    --dl-logs=1
+    --stop=1 

DistTestCore/BaseStarter.cs

@@ -1,15 +1,18 @@
-using Logging;
+using KubernetesWorkflow;
+using Logging;
 
 namespace DistTestCore
 {
     public class BaseStarter
     {
         protected readonly TestLifecycle lifecycle;
+        protected readonly WorkflowCreator workflowCreator;
         private Stopwatch? stopwatch;
 
-        public BaseStarter(TestLifecycle lifecycle)
+        public BaseStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
         {
             this.lifecycle = lifecycle;
+            this.workflowCreator = workflowCreator;
         }
 
         protected void LogStart(string msg)

DistTestCore/ByteSize.cs

@@ -38,11 +38,6 @@ namespace DistTestCore
     {
         private const long Kilo = 1024;
 
-        public static ByteSize Bytes(this long i)
-        {
-            return new ByteSize(i);
-        }
-
         public static ByteSize KB(this long i)
         {
             return new ByteSize(i * Kilo);
@@ -63,11 +58,6 @@ namespace DistTestCore
             return (i * Kilo).GB();
         }
 
-        public static ByteSize Bytes(this int i)
-        {
-            return new ByteSize(i);
-        }
-
         public static ByteSize KB(this int i)
         {
             return Convert.ToInt64(i).KB();

DistTestCore/Codex/CodexAccess.cs

@@ -4,11 +4,10 @@ using Utils;
 
 namespace DistTestCore.Codex
 {
-    public class CodexAccess : ILogHandler
+    public class CodexAccess
     {
         private readonly BaseLog log;
         private readonly ITimeSet timeSet;
-        private bool hasContainerCrashed;
 
         public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
         {
@@ -16,9 +15,6 @@ namespace DistTestCore.Codex
             Container = container;
             this.timeSet = timeSet;
             Address = address;
-            hasContainerCrashed = false;
-
-            if (container.CrashWatcher != null) container.CrashWatcher.Start(this);
         }
 
         public RunningContainer Container { get; }
@@ -88,36 +84,9 @@ namespace DistTestCore.Codex
             return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
         }
 
-        public string GetName()
-        {
-            return Container.Name;
-        }
-
         private Http Http()
         {
-            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
-        }
-
-        private void CheckContainerCrashed(HttpClient client)
-        {
-            if (hasContainerCrashed) throw new Exception("Container has crashed.");
-        }
-
-        public void Log(Stream crashLog)
-        {
-            var file = log.CreateSubfile();
-            log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
-
-            using var reader = new StreamReader(crashLog);
-            var line = reader.ReadLine();
-            while (line != null)
-            {
-                file.Write(line);
-                line = reader.ReadLine();
-            }
-
-            log.Log("Crash log successfully downloaded.");
-            hasContainerCrashed = true;
+            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name);
         }
     }
 }

DistTestCore/Codex/CodexApiTypes.cs

@@ -1,6 +1,5 @@
 using KubernetesWorkflow;
 using Logging;
-using Newtonsoft.Json;
 using Utils;
 
 namespace DistTestCore.Codex
@@ -61,16 +60,6 @@ namespace DistTestCore.Codex
     {
         public string version { get; set; } = string.Empty;
         public string revision { get; set; } = string.Empty;
-
-        public bool IsValid()
-        {
-            return !string.IsNullOrEmpty(version) && !string.IsNullOrEmpty(revision);
-        }
-
-        public override string ToString()
-        {
-            return JsonConvert.SerializeObject(this);
-        }
     }
 
     public class CodexDebugPeerResponse

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -5,8 +5,12 @@ namespace DistTestCore.Codex
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
-
+#if Arm64
+        public const string DockerImage = "codexstorage/nim-codex:sha-7227a4a";
+#else
+        //public const string DockerImage = "thatbenbierens/nim-codex:loopingyeah";
+        public const string DockerImage = "codexstorage/nim-codex:sha-7227a4a";
+#endif
         public const string MetricsPortTag = "metrics_port";
         public const string DiscoveryPortTag = "discovery-port";
 
@@ -14,12 +18,15 @@ namespace DistTestCore.Codex
         public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
         public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
 
-        public override string AppName => "codex";
-        public override string Image { get; }
+        public static string DockerImageOverride = string.Empty;
 
-        public CodexContainerRecipe()
+        protected override string Image
         {
-            Image = GetDockerImage();
+            get
+            {
+                if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride;
+                return DockerImage;
+            }
         }
 
         protected override void Initialize(StartupConfig startupConfig)
@@ -51,15 +58,7 @@ namespace DistTestCore.Codex
             {
                 AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
             }
-            if (config.BlockMaintenanceInterval != null)
-            {
-                AddEnvVar("CODEX_BLOCK_MI", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
-            }
-            if (config.BlockMaintenanceNumber != null)
-            {
-                AddEnvVar("CODEX_BLOCK_MN", config.BlockMaintenanceNumber.ToString()!);
-            }
-            if (config.MetricsMode != Metrics.MetricsMode.None)
+            if (config.MetricsEnabled)
             {
                 AddEnvVar("CODEX_METRICS", "true");
                 AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
@@ -93,12 +92,5 @@ namespace DistTestCore.Codex
             if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value;
             return Index;
         }
-
-        private string GetDockerImage()
-        {
-            var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
-            if (!string.IsNullOrEmpty(image)) return image;
-            return DefaultDockerImage;
-        }
     }
 }

DistTestCore/Codex/CodexDeployment.cs

@@ -5,25 +5,23 @@ namespace DistTestCore.Codex
 {
     public class CodexDeployment
     {
-        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, GrafanaStartInfo? grafanaStartInfo, DeploymentMetadata metadata)
+        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata)
         {
             GethStartResult = gethStartResult;
             CodexContainers = codexContainers;
             PrometheusContainer = prometheusContainer;
-            GrafanaStartInfo = grafanaStartInfo;
             Metadata = metadata;
         }
 
         public GethStartResult GethStartResult { get; }
         public RunningContainer[] CodexContainers { get; }
         public RunningContainer? PrometheusContainer { get; }
-        public GrafanaStartInfo? GrafanaStartInfo { get; }
         public DeploymentMetadata Metadata { get; }
     }
 
     public class DeploymentMetadata
     {
-        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
+        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration)
         {
             DeployDateTimeUtc = DateTime.UtcNow;
             KubeNamespace = kubeNamespace;
@@ -35,9 +33,6 @@ namespace DistTestCore.Codex
             MinPrice = minPrice;
             MaxCollateral = maxCollateral;
             MaxDuration = maxDuration;
-            BlockTTL = blockTTL;
-            BlockMI = blockMI;
-            BlockMN = blockMN;
         }
 
         public DateTime DeployDateTimeUtc { get; }
@@ -50,8 +45,5 @@ namespace DistTestCore.Codex
         public int MinPrice { get; }
         public int MaxCollateral { get; }
         public int MaxDuration { get; }
-        public int BlockTTL { get; }
-        public int BlockMI { get; }
-        public int BlockMN { get; }
     }
 }

DistTestCore/Codex/CodexStartupConfig.cs

@@ -1,5 +1,4 @@
 using DistTestCore.Marketplace;
-using DistTestCore.Metrics;
 using KubernetesWorkflow;
 
 namespace DistTestCore.Codex
@@ -15,11 +14,9 @@ namespace DistTestCore.Codex
         public Location Location { get; set; }
         public CodexLogLevel LogLevel { get; }
         public ByteSize? StorageQuota { get; set; }
-        public MetricsMode MetricsMode { get; set; }
+        public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
         public string? BootstrapSpr { get; set; }
         public int? BlockTTL { get; set; }
-        public TimeSpan? BlockMaintenanceInterval { get; set; }
-        public int? BlockMaintenanceNumber { get; set; }
     }
 }

DistTestCore/CodexNodeGroup.cs

@@ -14,13 +14,12 @@ namespace DistTestCore
     {
         private readonly TestLifecycle lifecycle;
 
-        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory)
         {
             this.lifecycle = lifecycle;
             Setup = setup;
             Containers = containers;
-            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
-            Version = new CodexDebugVersionResponse();
+            Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
         }
 
         public IOnlineCodexNode this[int index]
@@ -45,9 +44,8 @@ namespace DistTestCore
         }
 
         public CodexSetup Setup { get; private set; }
-        public RunningContainers[] Containers { get; private set; }
+        public RunningContainers Containers { get; private set; }
         public OnlineCodexNode[] Nodes { get; private set; }
-        public CodexDebugVersionResponse Version { get; private set; }
 
         public IEnumerator<IOnlineCodexNode> GetEnumerator()
         {
@@ -66,17 +64,14 @@ namespace DistTestCore
 
         public void EnsureOnline()
         {
-            foreach (var node in Nodes) node.EnsureOnlineGetVersionResponse();
-            var versionResponses = Nodes.Select(n => n.Version);
-
-            var first = versionResponses.First();
-            if (!versionResponses.All(v => v.version == first.version && v.revision == first.revision))
+            foreach (var node in Nodes)
             {
-                throw new Exception("Inconsistent version information received from one or more Codex nodes: " +
-                    string.Join(",", versionResponses.Select(v => v.ToString())));
+                var debugInfo = node.CodexAccess.GetDebugInfo();
+                var nodePeerId = debugInfo.id;
+                var nodeName = node.CodexAccess.Container.Name;
+                lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
+                lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
             }
-
-            Version = first;
         }
 
         private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)

DistTestCore/CodexSetup.cs

@@ -10,9 +10,6 @@ namespace DistTestCore
         ICodexSetup At(Location location);
         ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
         ICodexSetup WithStorageQuota(ByteSize storageQuota);
-        ICodexSetup WithBlockTTL(TimeSpan duration);
-        ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration);
-        ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks);
         ICodexSetup EnableMetrics();
         ICodexSetup EnableMarketplace(TestToken initialBalance);
         ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
@@ -53,27 +50,9 @@ namespace DistTestCore
             return this;
         }
 
-        public ICodexSetup WithBlockTTL(TimeSpan duration)
-        {
-            BlockTTL = Convert.ToInt32(duration.TotalSeconds);
-            return this;
-        }
-
-        public ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration)
-        {
-            BlockMaintenanceInterval = duration;
-            return this;
-        }
-
-        public ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks)
-        {
-            BlockMaintenanceNumber = numberOfBlocks;
-            return this;
-        }
-
         public ICodexSetup EnableMetrics()
         {
-            MetricsMode = Metrics.MetricsMode.Record;
+            MetricsEnabled = true;
             return this;
         }
 

DistTestCore/CodexStarter.cs

@@ -8,8 +8,8 @@ namespace DistTestCore
 {
     public class CodexStarter : BaseStarter
     {
-        public CodexStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
+        public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+            : base(lifecycle, workflowCreator)
         {
         }
 
@@ -22,7 +22,6 @@ namespace DistTestCore
             var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
 
             var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
-
             var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
 
             var metricAccessFactory = CollectMetrics(codexSetup, containers);
@@ -30,16 +29,9 @@ namespace DistTestCore
             var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
 
             var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
-            lifecycle.SetCodexVersion(group.Version);
-
-            var nl = Environment.NewLine;
-            var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
-            LogEnd($"Started {codexSetup.NumberOfNodes} nodes " +
-                $"of image '{containers.Containers().First().Recipe.Image}' " +
-                $"and version '{group.Version}'{nl}" +
-                podInfos);
+            var podInfo = group.Containers.RunningPod.PodInfo;
+            LogEnd($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers.First().Recipe.Image}' at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}");
             LogSeparator();
-
             return group;
         }
 
@@ -47,11 +39,7 @@ namespace DistTestCore
         {
             LogStart($"Stopping {group.Describe()}...");
             var workflow = CreateWorkflow();
-            foreach (var c in group.Containers)
-            {
-                StopCrashWatcher(c);
-                workflow.Stop(c);
-            }
+            workflow.Stop(group.Containers);
             RunningGroups.Remove(group);
             LogEnd("Stopped.");
         }
@@ -64,23 +52,17 @@ namespace DistTestCore
             RunningGroups.Clear();
         }
 
-        public void DownloadLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
+        public void DownloadLog(RunningContainer container, ILogHandler logHandler)
         {
             var workflow = CreateWorkflow();
-            workflow.DownloadContainerLog(container, logHandler, tailLines);
+            workflow.DownloadContainerLog(container, logHandler);
         }
 
-        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
+        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers containers)
         {
-            if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory();
+            if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
 
             var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
-
-            if (codexSetup.MetricsMode == MetricsMode.Dashboard)
-            {
-                lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup);
-            }
-
             return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
         }
 
@@ -93,66 +75,28 @@ namespace DistTestCore
             return startupConfig;
         }
 
-        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
+        private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
         {
-            var result = new List<RunningContainers>();
-            var recipe = new CodexContainerRecipe();
-            for (var i = 0; i < numberOfNodes; i++)
-            {
-                var workflow = CreateWorkflow();
-                var rc = workflow.Start(1, location, recipe, startupConfig);
-                CreateCrashWatcher(workflow, rc);
-                result.Add(rc);
-            }
-            return result.ToArray();
+            var workflow = CreateWorkflow();
+            return workflow.Start(numberOfNodes, location, new CodexContainerRecipe(), startupConfig);
         }
 
-        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
+        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
             RunningGroups.Add(group);
-
-            try
-            {
-                Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
-            }
-            catch
-            {
-                CodexNodesNotOnline(runningContainers);
-                throw;
-            }
-
+            Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
             return group;
         }
 
-        private void CodexNodesNotOnline(RunningContainers[] runningContainers)
-        {
-            Log("Codex nodes failed to start");
-            foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
-        }
-
         private StartupWorkflow CreateWorkflow()
         {
-            return lifecycle.WorkflowCreator.CreateWorkflow();
+            return workflowCreator.CreateWorkflow();
         }
 
         private void LogSeparator()
         {
             Log("----------------------------------------------------------------------------");
         }
-
-        private void CreateCrashWatcher(StartupWorkflow workflow, RunningContainers rc)
-        {
-            var c = rc.Containers.Single();
-            c.CrashWatcher = workflow.CreateCrashWatcher(c);
-        }
-
-        private void StopCrashWatcher(RunningContainers containers)
-        {
-            foreach (var c in containers.Containers)
-            {
-                c.CrashWatcher?.Stop();
-            }
-        }
     }
 }

DistTestCore/Configuration.cs

@@ -1,6 +1,5 @@
 using DistTestCore.Codex;
 using KubernetesWorkflow;
-using System.Net.NetworkInformation;
 using Utils;
 
 namespace DistTestCore
@@ -12,8 +11,7 @@ namespace DistTestCore
         private readonly bool logDebug;
         private readonly string dataFilesPath;
         private readonly CodexLogLevel codexLogLevel;
-        private readonly string k8sNamespacePrefix;
-        private static RunnerLocation? runnerLocation = null;
+        private readonly TestRunnerLocation runnerLocation;
 
         public Configuration()
         {
@@ -22,23 +20,23 @@ namespace DistTestCore
             logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
             dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
             codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
-            k8sNamespacePrefix = "ct-";
+            runnerLocation = ParseEnum.Parse<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
         }
 
-        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, string k8sNamespacePrefix)
+        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation)
         {
             this.kubeConfigFile = kubeConfigFile;
             this.logPath = logPath;
             this.logDebug = logDebug;
             this.dataFilesPath = dataFilesPath;
             this.codexLogLevel = codexLogLevel;
-            this.k8sNamespacePrefix = k8sNamespacePrefix;
+            this.runnerLocation = runnerLocation;
         }
 
         public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
         {
             return new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: k8sNamespacePrefix,
+                k8sNamespacePrefix: "ct-",
                 kubeConfigFile: kubeConfigFile,
                 operationTimeout: timeSet.K8sOperationTimeout(),
                 retryDelay: timeSet.WaitForK8sServiceDelay()
@@ -60,14 +58,14 @@ namespace DistTestCore
             return codexLogLevel;
         }
 
+        public TestRunnerLocation GetTestRunnerLocation()
+        {
+            return runnerLocation;
+        }
+
         public Address GetAddress(RunningContainer container)
         {
-            if (runnerLocation == null)
-            {
-                runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container);
-            }
-            
-            if (runnerLocation == RunnerLocation.InternalToCluster)
+            if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster)
             {
                 return container.ClusterInternalAddress;
             }
@@ -89,56 +87,9 @@ namespace DistTestCore
         }
     }
 
-    public enum RunnerLocation
+    public enum TestRunnerLocation
     {
         ExternalToCluster,
         InternalToCluster,
     }
-
-    public static class RunnerLocationUtils
-    {
-        private static bool alreadyDidThat = false;
-
-        public static RunnerLocation DetermineRunnerLocation(RunningContainer container)
-        {
-            // We want to be sure we don't ping more often than strictly necessary.
-            // If we have already determined the location during this application
-            // lifetime, don't do it again.
-            if (alreadyDidThat) throw new Exception("We already did that.");
-            alreadyDidThat = true;
-
-            if (PingHost(container.Pod.PodInfo.Ip))
-            {
-                return RunnerLocation.InternalToCluster;
-            }
-            if (PingHost(Format(container.ClusterExternalAddress)))
-            {
-                return RunnerLocation.ExternalToCluster;
-            }
-
-            throw new Exception("Unable to determine runner location.");
-        }
-
-        private static string Format(Address host)
-        {
-            return host.Host
-                .Replace("http://", "")
-                .Replace("https://", "");
-        }
-
-        private static bool PingHost(string host)
-        {
-            try
-            {
-                using var pinger = new Ping();
-                PingReply reply = pinger.Send(host);
-                return reply.Status == IPStatus.Success;
-            }
-            catch (PingException)
-            {
-            }
-
-            return false;
-        }
-    }
 }

DistTestCore/DistTest.cs

@@ -13,11 +13,9 @@ namespace DistTestCore
     [Parallelizable(ParallelScope.All)]
     public abstract class DistTest
     {
-        private const string TestsType = "dist-tests";
         private readonly Configuration configuration = new Configuration();
         private readonly Assembly[] testAssemblies;
         private readonly FixtureLog fixtureLog;
-        private readonly StatusLog statusLog;
         private readonly object lifecycleLock = new object();
         private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
 
@@ -26,28 +24,25 @@ namespace DistTestCore
             var assemblies = AppDomain.CurrentDomain.GetAssemblies();
             testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
 
-            var logConfig = configuration.GetLogConfig();
-            var startTime = DateTime.UtcNow;
-            fixtureLog = new FixtureLog(logConfig, startTime);
-            statusLog = new StatusLog(logConfig, startTime);
+            fixtureLog = new FixtureLog(configuration.GetLogConfig());
+
+            PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
+            PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
         }
 
+        public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; }
+        public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; }
+
         [OneTimeSetUp]
         public void GlobalSetup()
         {
-            fixtureLog.Log($"Codex Distributed Tests are starting...");
-            fixtureLog.Log($"Codex image: '{new CodexContainerRecipe().Image}'");
-            fixtureLog.Log($"CodexContracts image: '{new CodexContractsContainerRecipe().Image}'");
-            fixtureLog.Log($"Prometheus image: '{new PrometheusContainerRecipe().Image}'");
-            fixtureLog.Log($"Geth image: '{new GethContainerRecipe().Image}'");
-
             // Previous test run may have been interrupted.
             // Begin by cleaning everything up.
             try
             {
                 Stopwatch.Measure(fixtureLog, "Global setup", () =>
                 {
-                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), new PodLabels(TestsType, null!), string.Empty);
+                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
                     wc.CreateWorkflow().DeleteAllResources();
                 });                
             }
@@ -59,6 +54,9 @@ namespace DistTestCore
             }
 
             fixtureLog.Log("Global setup cleanup successful");
+            fixtureLog.Log($"Codex image: '{CodexContainerRecipe.DockerImage}'");
+            fixtureLog.Log($"Prometheus image: '{PrometheusContainerRecipe.DockerImage}'");
+            fixtureLog.Log($"Geth image: '{GethContainerRecipe.DockerImage}'");
         }
 
         [SetUp]
@@ -169,21 +167,6 @@ namespace DistTestCore
             GetTestLog().Debug(msg);
         }
 
-        public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers()
-        {
-            return new PeerConnectionTestHelpers(GetTestLog());
-        }
-
-        public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers()
-        {
-            return new PeerDownloadTestHelpers(GetTestLog(), Get().FileManager);
-        }
-
-        public void Measure(string name, Action action)
-        {
-            Stopwatch.Measure(Get().Log, name, action);
-        }
-
         protected CodexSetup CreateCodexSetup(int numberOfNodes)
         {
             return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
@@ -200,14 +183,11 @@ namespace DistTestCore
         private void CreateNewTestLifecycle()
         {
             var testName = GetCurrentTestName();
-            fixtureLog.WriteLogTag();
             Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
             {
                 lock (lifecycleLock)
                 {
-                    var testNamespace = Guid.NewGuid().ToString();
-                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), TestsType, testNamespace);
-                    lifecycles.Add(testName, lifecycle);
+                    lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
                 }
             });
         }
@@ -215,10 +195,7 @@ namespace DistTestCore
         private void DisposeTestLifecycle()
         {
             var lifecycle = Get();
-            var testResult = GetTestResult();
-            var testDuration = lifecycle.GetTestDuration();
-            fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
-            statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetApplicationIds());
+            fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
             Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
             {
                 lifecycle.Log.EndTest();

DistTestCore/DistTestCore.csproj

@@ -10,14 +10,6 @@
   <PropertyGroup Condition="'$(IsArm64)'=='true'">
     <DefineConstants>Arm64</DefineConstants>
   </PropertyGroup>
-  <ItemGroup>
-    <None Remove="Metrics\dashboard.json" />
-  </ItemGroup>
-  <ItemGroup>
-    <EmbeddedResource Include="Metrics\dashboard.json">
-      <CopyToOutputDirectory>Never</CopyToOutputDirectory>
-    </EmbeddedResource>
-  </ItemGroup>
 
   <ItemGroup>
 	  <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />

DistTestCore/GethStarter.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Marketplace;
+using KubernetesWorkflow;
 
 namespace DistTestCore
 {
@@ -7,13 +8,13 @@ namespace DistTestCore
         private readonly MarketplaceNetworkCache marketplaceNetworkCache;
         private readonly GethCompanionNodeStarter companionNodeStarter;
 
-        public GethStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
+        public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+            : base(lifecycle, workflowCreator)
         {
             marketplaceNetworkCache = new MarketplaceNetworkCache(
-                new GethBootstrapNodeStarter(lifecycle),
-                new CodexContractsStarter(lifecycle));
-            companionNodeStarter = new GethCompanionNodeStarter(lifecycle);
+                new GethBootstrapNodeStarter(lifecycle, workflowCreator),
+                new CodexContractsStarter(lifecycle, workflowCreator));
+            companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator);
         }
 
         public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup)

DistTestCore/GrafanaStarter.cs

@@ -1,192 +0,0 @@
-using DistTestCore.Metrics;
-using IdentityModel.Client;
-using KubernetesWorkflow;
-using Newtonsoft.Json;
-using System.Reflection;
-
-namespace DistTestCore
-{
-    public class GrafanaStarter : BaseStarter
-    {
-        private const string StorageQuotaThresholdReplaceToken = "\"<CODEX_STORAGEQUOTA>\"";
-        private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"<CODEX_BYTESUSED_SOFTMAX>\"";
-
-        public GrafanaStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
-        {
-        }
-
-        public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup)
-        {
-            LogStart($"Starting dashboard server");
-
-            var grafanaContainer = StartGrafanaContainer();
-            var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer);
-
-            var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth);
-
-            Log("Connecting datasource...");
-            AddDataSource(http, prometheusContainer);
-
-            Log("Uploading dashboard configurations...");
-            var jsons = ReadEachDashboardJsonFile(codexSetup);
-            var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
-
-            LogEnd("Dashboard server started.");
-
-            return new GrafanaStartInfo(dashboardUrls, grafanaContainer);
-        }
-
-        private RunningContainer StartGrafanaContainer()
-        {
-            var startupConfig = new StartupConfig();
-
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
-            var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig);
-            if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created.");
-
-            return grafanaContainers.Containers.First();
-        }
-
-        private void AddBasicAuth(HttpClient client)
-        {
-            client.SetBasicAuthentication(
-                GrafanaContainerRecipe.DefaultAdminUser,
-                GrafanaContainerRecipe.DefaultAdminPassword);
-        }
-
-        private static void AddDataSource(Http http, RunningContainer prometheusContainer)
-        {
-            var prometheusAddress = prometheusContainer.ClusterExternalAddress;
-            var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port;
-            var response = http.HttpPostJson<GrafanaDataSourceRequest, GrafanaDataSourceResponse>("datasources", new GrafanaDataSourceRequest
-            {
-                uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb",
-                name = "CodexPrometheus",
-                type = "prometheus",
-                url = prometheusUrl,
-                access = "proxy",
-                basicAuth = false,
-                jsonData = new GrafanaDataSourceJsonData
-                {
-                    httpMethod = "POST"
-                }
-            });
-
-            if (response.message != "Datasource added")
-            {
-                throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message);
-            }
-        }
-
-        public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson)
-        {
-            var request = GetDashboardCreateRequest(dashboardJson);
-            var response = http.HttpPostString("dashboards/db", request);
-            var jsonResponse = JsonConvert.DeserializeObject<GrafanaPostDashboardResponse>(response);
-            if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard.");
-
-            var grafanaAddress = grafanaContainer.ClusterExternalAddress;
-            return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
-        }
-
-        private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup)
-        {
-            var assembly = Assembly.GetExecutingAssembly();
-            var resourceNames = new[]
-            {
-                "DistTestCore.Metrics.dashboard.json"
-            };
-
-            return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray();
-        }
-
-        private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup)
-        {
-            using var stream = assembly.GetManifestResourceStream(resourceName);
-            if (stream == null) throw new Exception("Unable to find resource " + resourceName);
-            using var reader = new StreamReader(stream);
-            return ApplyReplacements(reader.ReadToEnd(), codexSetup);
-        }
-
-        private static string ApplyReplacements(string input, CodexSetup codexSetup)
-        {
-            var quotaString = GetQuotaString(codexSetup);
-            var softMaxString = GetSoftMaxString(codexSetup);
-
-            return input
-                .Replace(StorageQuotaThresholdReplaceToken, quotaString)
-                .Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString);
-        }
-
-        private static string GetQuotaString(CodexSetup codexSetup)
-        {
-            return GetCodexStorageQuotaInBytes(codexSetup).ToString();
-        }
-
-        private static string GetSoftMaxString(CodexSetup codexSetup)
-        {
-            var quota = GetCodexStorageQuotaInBytes(codexSetup);
-            var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing.
-            return softMax.ToString();
-        }
-
-        private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup)
-        {
-            if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes;
-
-            // Codex default: 8GB
-            return 8.GB().SizeInBytes;
-        }
-
-        private static string GetDashboardCreateRequest(string dashboardJson)
-        {
-            return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}";
-        }
-    }
-
-    public class GrafanaStartInfo
-    {
-        public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container)
-        {
-            DashboardUrls = dashboardUrls;
-            Container = container;
-        }
-
-        public string[] DashboardUrls { get; }
-        public RunningContainer Container { get; }
-    }
-
-    public class GrafanaDataSourceRequest
-    {
-        public string uid { get; set; } = string.Empty;
-        public string name { get; set; } = string.Empty;
-        public string type { get; set; } = string.Empty;
-        public string url { get; set; } = string.Empty;
-        public string access { get; set; } = string.Empty;
-        public bool basicAuth { get; set; }
-        public GrafanaDataSourceJsonData jsonData { get; set; } = new();
-    }
-
-    public class GrafanaDataSourceResponse
-    {
-        public int id { get; set; }
-        public string message { get; set; } = string.Empty;
-        public string name { get; set; } = string.Empty;
-    }
-
-    public class GrafanaDataSourceJsonData
-    {
-        public string httpMethod { get; set; } = string.Empty;
-    }
-
-    public class GrafanaPostDashboardResponse
-    {
-        public int id { get; set; }
-        public string slug { get; set; } = string.Empty;
-        public string status { get; set; } = string.Empty;
-        public string uid { get; set; } = string.Empty;
-        public string url { get; set; } = string.Empty;
-        public int version { get; set; }
-    }
-}

DistTestCore/Helpers/FullConnectivityHelper.cs

@@ -1,217 +0,0 @@
-using DistTestCore.Codex;
-using Logging;
-using NUnit.Framework;
-using Utils;
-
-namespace DistTestCore.Helpers
-{
-    public interface IFullConnectivityImplementation
-    {
-        string Description();
-        string ValidateEntry(FullConnectivityHelper.Entry entry, FullConnectivityHelper.Entry[] allEntries);
-        FullConnectivityHelper.PeerConnectionState Check(FullConnectivityHelper.Entry from, FullConnectivityHelper.Entry to);
-    }
-
-    public class FullConnectivityHelper
-    {
-        private static string Nl = Environment.NewLine;
-        private readonly BaseLog log;
-        private readonly IFullConnectivityImplementation implementation;
-
-        public FullConnectivityHelper(BaseLog log, IFullConnectivityImplementation implementation)
-        {
-            this.log = log;
-            this.implementation = implementation;
-        }
-
-        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
-        {
-            AssertFullyConnected(nodes.ToArray());
-        }
-
-        private void AssertFullyConnected(CodexAccess[] nodes)
-        {
-            Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
-            var entries = CreateEntries(nodes);
-            var pairs = CreatePairs(entries);
-
-            RetryWhilePairs(pairs, () =>
-            {
-                CheckAndRemoveSuccessful(pairs);
-            });
-
-            if (pairs.Any())
-            {
-                var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
-
-                Log($"Connections failed:{Nl}{pairDetails}");
-
-                Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
-            }
-            else
-            {
-                Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
-            }
-        }
-
-        private static void RetryWhilePairs(List<Pair> pairs, Action action)
-        {
-            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2);
-            while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
-            {
-                action();
-
-                Time.Sleep(TimeSpan.FromSeconds(2));
-            }
-        }
-
-        private void CheckAndRemoveSuccessful(List<Pair> pairs)
-        {
-            // For large sets, don't try and do all of them at once.
-            var selectedPair = pairs.Take(20).ToArray();
-            var pairDetails = new List<string>();
-
-            foreach (var pair in selectedPair)
-            {
-                pair.Check();
-
-                if (pair.Success)
-                {
-                    pairDetails.AddRange(pair.GetResultMessages());
-                    pairs.Remove(pair);
-                }
-            }
-
-            Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
-        }
-
-        private Entry[] CreateEntries(CodexAccess[] nodes)
-        {
-            var entries = nodes.Select(n => new Entry(n)).ToArray();
-
-            var errors = entries
-                            .Select(e => implementation.ValidateEntry(e, entries))
-                            .Where(s => !string.IsNullOrEmpty(s))
-                            .ToArray();
-
-            if (errors.Any())
-            {
-                Assert.Fail("Some node entries failed to validate: " + string.Join(Nl, errors));
-            }
-
-            return entries;
-        }
-
-        private List<Pair> CreatePairs(Entry[] entries)
-        {
-            return CreatePairsIterator(entries).ToList();
-        }
-
-        private IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
-        {
-            for (var x = 0; x < entries.Length; x++)
-            {
-                for (var y = x + 1; y < entries.Length; y++)
-                {
-                    yield return new Pair(implementation, entries[x], entries[y]);
-                }
-            }
-        }
-
-        private void Log(string msg)
-        {
-            log.Log(msg);
-        }
-
-        public class Entry
-        {
-            public Entry(CodexAccess node)
-            {
-                Node = node;
-                Response = node.GetDebugInfo();
-            }
-
-            public CodexAccess Node { get; }
-            public CodexDebugResponse Response { get; }
-
-            public override string ToString()
-            {
-                if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
-                return Response.id;
-            }
-        }
-
-        public enum PeerConnectionState
-        {
-            Unknown,
-            Connection,
-            NoConnection,
-        }
-
-        public class Pair
-        {
-            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
-            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
-            private readonly IFullConnectivityImplementation implementation;
-
-            public Pair(IFullConnectivityImplementation implementation, Entry a, Entry b)
-            {
-                this.implementation = implementation;
-                A = a;
-                B = b;
-            }
-
-            public Entry A { get; }
-            public Entry B { get; }
-            public PeerConnectionState AKnowsB { get; private set; }
-            public PeerConnectionState BKnowsA { get; private set; }
-            public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
-            public bool Inconclusive { get { return AKnowsB == PeerConnectionState.Unknown || BKnowsA == PeerConnectionState.Unknown; } }
-
-            public void Check()
-            {
-                aToBTime = Measure(() => AKnowsB = Check(A, B));
-                bToATime = Measure(() => BKnowsA = Check(B, A));
-            }
-
-            public override string ToString()
-            {
-                return $"[{string.Join(",", GetResultMessages())}]";
-            }
-
-            public string[] GetResultMessages()
-            {
-                var aName = A.ToString();
-                var bName = B.ToString();
-
-                return new[]
-                {
-                    $"[{aName} --> {bName}] = {AKnowsB} ({aToBTime.TotalSeconds} seconds)",
-                    $"[{aName} <-- {bName}] = {BKnowsA} ({bToATime.TotalSeconds} seconds)"
-                };
-            }
-
-            private static TimeSpan Measure(Action action)
-            {
-                var start = DateTime.UtcNow;
-                action();
-                return DateTime.UtcNow - start;
-            }
-
-            private PeerConnectionState Check(Entry from, Entry to)
-            {
-                Thread.Sleep(10);
-
-                try
-                {
-                    return implementation.Check(from, to);
-                }
-                catch
-                {
-                    // Didn't get a conclusive answer. Try again later.
-                    return PeerConnectionState.Unknown;
-                }
-            }
-        }
-    }
-}

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -1,71 +1,257 @@
 using DistTestCore.Codex;
-using Logging;
-using static DistTestCore.Helpers.FullConnectivityHelper;
+using NUnit.Framework;
+using Utils;
 
 namespace DistTestCore.Helpers
 {
-    public class PeerConnectionTestHelpers : IFullConnectivityImplementation
+    public class PeerConnectionTestHelpers
     {
-        private readonly FullConnectivityHelper helper;
+        private readonly Random random = new Random();
+        private readonly DistTest test;
 
-        public PeerConnectionTestHelpers(BaseLog log)
+        public PeerConnectionTestHelpers(DistTest test)
         {
-            helper = new FullConnectivityHelper(log, this);
+            this.test = test;
         }
 
         public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
         {
-            AssertFullyConnected(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess));
-        }
+            var n = nodes.ToArray();
 
-        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
-        {
-            helper.AssertFullyConnected(nodes);
-        }
+            AssertFullyConnected(n);
 
-        public string Description()
-        {
-            return "Peer Discovery";
-        }
-
-        public string ValidateEntry(Entry entry, Entry[] allEntries)
-        {
-            var result = string.Empty;
-            foreach (var peer in entry.Response.table.nodes)
+            for (int i = 0; i < 5; i++)
             {
-                var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
-                if (expected != peer.address)
+                Time.Sleep(TimeSpan.FromSeconds(30));
+                AssertFullyConnected(n);
+            }
+        }
+
+        private void AssertFullyConnected(IOnlineCodexNode[] nodes)
+        {
+            test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            var entries = CreateEntries(nodes);
+            var pairs = CreatePairs(entries);
+
+            RetryWhilePairs(pairs, () =>
+            {
+                CheckAndRemoveSuccessful(pairs);
+            });
+
+            if (pairs.Any())
+            {
+                test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
+                Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
+                test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
+            }
+            else
+            {
+                test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
+            }
+        }
+
+        private static void RetryWhilePairs(List<Pair> pairs, Action action)
+        {
+            var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(30);
+            while (pairs.Any() && timeout > DateTime.UtcNow)
+            {
+                action();
+
+                if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(2));
+            }
+        }
+
+        private void CheckAndRemoveSuccessful(List<Pair> pairs)
+        {
+            var checkTasks = pairs.Select(p => Task.Run(() =>
+            {
+                ApplyRandomDelay();
+                p.Check();
+            })).ToArray();
+
+            Task.WaitAll(checkTasks);
+
+            foreach (var pair in pairs.ToArray())
+            {
+                if (pair.Success)
                 {
-                    result += $"Node:{entry.Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'. ";
+                    test.Log(pair.GetMessage());
+                    pairs.Remove(pair);
                 }
             }
-            return result;
         }
 
-        public PeerConnectionState Check(Entry from, Entry to)
+        private static Entry[] CreateEntries(IOnlineCodexNode[] nodes)
         {
-            var peerId = to.Response.id;
+            var entries = nodes.Select(n => new Entry(n)).ToArray();
+            var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
 
-            var response = from.Node.GetDebugPeer(peerId);
-            if (!response.IsPeerFound)
+            if (incorrectDiscoveryEndpoints.Any())
             {
-                return PeerConnectionState.NoConnection;
+                Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
+                    string.Join(Environment.NewLine, incorrectDiscoveryEndpoints));
             }
-            if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
-            {
-                return PeerConnectionState.Connection;
-            }
-            return PeerConnectionState.Unknown;
+
+            return entries;
         }
 
-        private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
+        private static List<Pair> CreatePairs(Entry[] entries)
         {
-            var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
-            if (peer == null) return $"peerId: {node.peerId} is not known.";
+            return CreatePairsIterator(entries).ToList();
+        }
 
-            var ip = peer.Node.Container.Pod.PodInfo.Ip;
-            var discPort = peer.Node.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
-            return $"{ip}:{discPort.Number}";
+        private static IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
+        {
+            for (var x = 0; x < entries.Length; x++)
+            {
+                for (var y = x + 1; y < entries.Length; y++)
+                {
+                    yield return new Pair(entries[x], entries[y]);
+                }
+            }
+        }
+
+        private void ApplyRandomDelay()
+        {
+            // Calling all the nodes all at the same time is not exactly nice.
+            Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000)));
+        }
+
+        public class Entry
+        {
+            public Entry(IOnlineCodexNode node)
+            {
+                Node = node;
+                Response = node.GetDebugInfo();
+            }
+
+            public IOnlineCodexNode Node { get; }
+            public CodexDebugResponse Response { get; }
+
+            public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
+            {
+                foreach (var peer in Response.table.nodes)
+                {
+                    var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
+                    if (expected != peer.address)
+                    {
+                        yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'";
+                    }
+                }
+            }
+
+            public override string ToString()
+            {
+                if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
+                return Response.id;
+            }
+
+            private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
+            {
+                var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
+                if (peer == null) return $"peerId: {node.peerId} is not known.";
+
+                var n = (OnlineCodexNode)peer.Node;
+                var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
+                var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
+                return $"{ip}:{discPort.Number}";
+            }
+        }
+
+        public enum PeerConnectionState
+        {
+            Unknown,
+            Connection,
+            NoConnection,
+        }
+
+        public class Pair
+        {
+            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
+            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
+
+            public Pair(Entry a, Entry b)
+            {
+                A = a;
+                B = b;
+            }
+
+            public Entry A { get; }
+            public Entry B { get; }
+            public PeerConnectionState AKnowsB { get; private set; }
+            public PeerConnectionState BKnowsA { get; private set; }
+            public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
+
+            public void Check()
+            {
+                aToBTime = Measure(() => AKnowsB = Knows(A, B));
+                bToATime = Measure(() => BKnowsA = Knows(B, A));
+            }
+
+            public string GetMessage()
+            {
+                return GetResultMessage() + GetTimePostfix();
+            }
+
+            public override string ToString()
+            {
+                return $"[{GetMessage()}]";
+            }
+
+            private string GetResultMessage()
+            {
+                var aName = A.ToString();
+                var bName = B.ToString();
+
+                if (Success)
+                {
+                    return $"{aName} and {bName} know each other.";
+                }
+
+                return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}";
+            }
+
+            private string GetTimePostfix()
+            {
+                var aName = A.ToString();
+                var bName = B.ToString();
+
+                return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
+            }
+
+            private static TimeSpan Measure(Action action)
+            {
+                var start = DateTime.UtcNow;
+                action();
+                return DateTime.UtcNow - start;
+            }
+
+            private PeerConnectionState Knows(Entry a, Entry b)
+            {
+                lock (a)
+                {
+                    var peerId = b.Response.id;
+
+                    try
+                    {
+                        var response = a.Node.GetDebugPeer(peerId);
+                        if (!response.IsPeerFound)
+                        {
+                            return PeerConnectionState.NoConnection;
+                        }
+                        if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
+                        {
+                            return PeerConnectionState.Connection;
+                        }
+                    }
+                    catch
+                    {
+                    }
+
+                    // Didn't get a conclusive answer. Try again later.
+                    return PeerConnectionState.Unknown;
+                }
+            }
         }
     }
 }

DistTestCore/Helpers/PeerDownloadTestHelpers.cs

@@ -1,88 +1,75 @@
 using DistTestCore.Codex;
-using Logging;
-using static DistTestCore.Helpers.FullConnectivityHelper;
+using NUnit.Framework;
 
 namespace DistTestCore.Helpers
 {
-    public class PeerDownloadTestHelpers : IFullConnectivityImplementation
+    public class PeerDownloadTestHelpers
     {
-        private readonly FullConnectivityHelper helper;
-        private readonly BaseLog log;
-        private readonly FileManager fileManager;
-        private ByteSize testFileSize;
+        private readonly DistTest test;
 
-        public PeerDownloadTestHelpers(BaseLog log, FileManager fileManager)
+        public PeerDownloadTestHelpers(DistTest test)
         {
-            helper = new FullConnectivityHelper(log, this);
-            testFileSize = 1.MB();
-            this.log = log;
-            this.fileManager = fileManager;
+            this.test = test;
         }
 
         public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
         {
-            AssertFullDownloadInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess), testFileSize);
-        }
+            test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            var start = DateTime.UtcNow;
 
-        public void AssertFullDownloadInterconnectivity(IEnumerable<CodexAccess> nodes, ByteSize testFileSize)
-        {
-            this.testFileSize = testFileSize;
-            helper.AssertFullyConnected(nodes);
-        }
-
-        public string Description()
-        {
-            return "Download Connectivity";
-        }
-
-        public string ValidateEntry(Entry entry, Entry[] allEntries)
-        {
-            return string.Empty;
-        }
-
-        public PeerConnectionState Check(Entry from, Entry to)
-        {
-            fileManager.PushFileSet();
-            var expectedFile = GenerateTestFile(from.Node, to.Node);
-
-            using var uploadStream = File.OpenRead(expectedFile.Filename);
-            var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(uploadStream));
-
-            try
+            foreach (var node in nodes)
             {
-                var downloadedFile = Stopwatch.Measure(log, "Download", () => DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded"));
+                var uploader = node;
+                var downloaders = nodes.Where(n => n != uploader).ToArray();
+
+                test.ScopedTestFiles(() =>
+                {
+                    PerformTest(uploader, downloaders, testFileSize);
+                });
+            }
+            
+            test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
+            var timeTaken = DateTime.UtcNow - start;
+
+            AssertTimePerMB(timeTaken, nodes.Count(), testFileSize);
+        }
+
+        private void AssertTimePerMB(TimeSpan timeTaken, int numberOfNodes, ByteSize size)
+        {
+            var numberOfDownloads = numberOfNodes * (numberOfNodes - 1);
+            var timePerDownload = timeTaken / numberOfDownloads;
+            float sizeInMB = size.ToMB();
+            var timePerMB = timePerDownload / sizeInMB;
+
+            test.Log($"Performed {numberOfDownloads} downloads of {size} in {timeTaken.TotalSeconds} seconds, for an average of {timePerMB.TotalSeconds} seconds per MB.");
+
+            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
+        }
+
+        private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
+        {
+            // Generate 1 test file per downloader.
+            var files = downloaders.Select(d => GenerateTestFile(uploader, d, testFileSize)).ToArray();
+
+            // Upload all the test files to the uploader.
+            var contentIds = files.Select(uploader.UploadFile).ToArray();
+
+            // Each downloader should retrieve its own test file.
+            for (var i = 0; i < downloaders.Length; i++)
+            {
+                var expectedFile = files[i];
+                var downloadedFile = downloaders[i].DownloadContent(contentIds[i], $"{expectedFile.Label}DOWNLOADED");
+
                 expectedFile.AssertIsEqual(downloadedFile);
-                return PeerConnectionState.Connection;
             }
-            catch
-            {
-                // Should an exception occur during the download or file-content assertion,
-                // We consider that as no-connection for the purpose of this test.
-                return PeerConnectionState.NoConnection;
-            }
-            finally
-            {
-                fileManager.PopFileSet();
-            }
-
-            // Should an exception occur during upload, then this try is inconclusive and we try again next loop.
         }
 
-        private TestFile DownloadFile(CodexAccess node, string contentId, string label)
-        {
-            var downloadedFile = fileManager.CreateEmptyTestFile(label);
-            using var downloadStream = File.OpenWrite(downloadedFile.Filename);
-            using var stream = node.DownloadFile(contentId);
-            stream.CopyTo(downloadStream);
-            return downloadedFile;
-        }
-
-        private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader)
+        private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader, ByteSize testFileSize)
         {
             var up = uploader.GetName().Replace("<", "").Replace(">", "");
             var down = downloader.GetName().Replace("<", "").Replace(">", "");
-            var label = $"~from:{up}-to:{down}~";
-            return fileManager.GenerateTestFile(testFileSize, label);
+            var label = $"FROM{up}TO{down}";
+            return test.GenerateTestFile(testFileSize, label);
         }
     }
 }

DistTestCore/Http.cs

@@ -12,21 +12,14 @@ namespace DistTestCore
         private readonly ITimeSet timeSet;
         private readonly Address address;
         private readonly string baseUrl;
-        private readonly Action<HttpClient> onClientCreated;
         private readonly string? logAlias;
 
         public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
-            : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
-        {
-        }
-
-        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
         {
             this.log = log;
             this.timeSet = timeSet;
             this.address = address;
             this.baseUrl = baseUrl;
-            this.onClientCreated = onClientCreated;
             this.logAlias = logAlias;
             if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
             if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
@@ -73,22 +66,6 @@ namespace DistTestCore
             }, $"HTTP-POST-JSON: {route}");
         }
 
-        public string HttpPostString(string route, string body)
-        {
-            return Retry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                Log(url, body);
-                var content = new StringContent(body);
-                content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
-                var result = Time.Wait(client.PostAsync(url, content));
-                var str = Time.Wait(result.Content.ReadAsStringAsync());
-                Log(url, str);
-                return str;
-            }, $"HTTP-POST-STRING: {route}");
-        }
-
         public string HttpPostStream(string route, Stream stream)
         {
             return Retry(() =>
@@ -99,7 +76,7 @@ namespace DistTestCore
                 var content = new StreamContent(stream);
                 content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
                 var response = Time.Wait(client.PostAsync(url, content));
-                var str = Time.Wait(response.Content.ReadAsStringAsync());
+                var str =Time.Wait(response.Content.ReadAsStringAsync());
                 Log(url, str);
                 return str;
             }, $"HTTP-POST-STREAM: {route}");
@@ -155,12 +132,7 @@ namespace DistTestCore
         {
             var client = new HttpClient();
             client.Timeout = timeSet.HttpCallTimeout();
-            onClientCreated(client);
             return client;
         }
-
-        private static void DoNothing(HttpClient client)
-        {
-        }
     }
 }

DistTestCore/Logs/DownloadedLog.cs

@@ -6,8 +6,6 @@ namespace DistTestCore.Logs
     public interface IDownloadedLog
     {
         void AssertLogContains(string expectedString);
-        string[] FindLinesThatContain(params string[] tags);
-        void DeleteFile();
     }
 
     public class DownloadedLog : IDownloadedLog
@@ -35,30 +33,5 @@ namespace DistTestCore.Logs
 
             Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
         }
-
-        public string[] FindLinesThatContain(params string[] tags)
-        {
-            var result = new List<string>();
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (tags.All(line.Contains))
-                {
-                    result.Add(line);
-                }
-
-                line = streamReader.ReadLine();
-            }
-
-            return result.ToArray();
-        }
-
-        public void DeleteFile()
-        {
-            File.Delete(logFile.FullFilename);
-        }
     }
 }

DistTestCore/Marketplace/CodexContractsContainerRecipe.cs

@@ -4,11 +4,15 @@ namespace DistTestCore.Marketplace
 {
     public class CodexContractsContainerRecipe : ContainerRecipeFactory
     {
-        public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
-        public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
+        #if Arm64
+            public const string DockerImage = "emizzle/codex-contracts-deployment:latest";
+        #else
+            public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint2";
+        #endif
+        public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json";
+        public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json";
 
-        public override string AppName => "codex-contracts";
-        public override string Image => "codexstorage/codex-contracts-eth:latest-dist-tests";
+        protected override string Image => DockerImage;
 
         protected override void Initialize(StartupConfig startupConfig)
         {

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -6,8 +6,8 @@ namespace DistTestCore.Marketplace
     public class CodexContractsStarter : BaseStarter
     {
 
-        public CodexContractsStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
+        public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+            : base(lifecycle, workflowCreator)
         {
         }
 
@@ -15,7 +15,7 @@ namespace DistTestCore.Marketplace
         {
             LogStart("Deploying Codex Marketplace...");
 
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var workflow = workflowCreator.CreateWorkflow();
             var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]);
 
             var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig);
@@ -25,7 +25,7 @@ namespace DistTestCore.Marketplace
             WaitUntil(() =>
             {
                 var logHandler = new ContractsReadyLogHandler(Debug);
-                workflow.DownloadContainerLog(container, logHandler, null);
+                workflow.DownloadContainerLog(container, logHandler);
                 return logHandler.Found;
             });
             Log("Contracts deployed. Extracting addresses...");
@@ -91,13 +91,8 @@ namespace DistTestCore.Marketplace
         {
             debug(line);
             if (line.Contains(RequiredCompiledString)) SeenCompileString = true;
-            if (line.Contains(ReadyString))
-            {
-                if (!SeenCompileString) throw new Exception("CodexContracts deployment failed. " +
-                    "Solidity files not compiled before process exited.");
 
-                Found = true;
-            }
+            if (SeenCompileString && line.Contains(ReadyString)) Found = true;
         }
     }
 }

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -80,7 +80,7 @@ namespace DistTestCore.Marketplace
         private string FetchPubKey()
         {
             var enodeFinder = new PubKeyFinder(s => log.Debug(s));
-            workflow.DownloadContainerLog(container, enodeFinder, null);
+            workflow.DownloadContainerLog(container, enodeFinder);
             return enodeFinder.GetPubKey();
         }
 

DistTestCore/Marketplace/GethBootstrapNodeStarter.cs

@@ -4,8 +4,8 @@ namespace DistTestCore.Marketplace
 {
     public class GethBootstrapNodeStarter : BaseStarter
     {
-        public GethBootstrapNodeStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
+        public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+            : base(lifecycle, workflowCreator)
         {
         }
 
@@ -14,7 +14,7 @@ namespace DistTestCore.Marketplace
             LogStart("Starting Geth bootstrap node...");
             var startupConfig = CreateBootstrapStartupConfig();
             
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var workflow = workflowCreator.CreateWorkflow();
             var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
             var bootstrapContainer = containers.Containers[0];

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -7,8 +7,8 @@ namespace DistTestCore.Marketplace
     {
         private int companionAccountIndex = 0;
 
-        public GethCompanionNodeStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
+        public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+            : base(lifecycle, workflowCreator)
         {
         }
 
@@ -18,7 +18,7 @@ namespace DistTestCore.Marketplace
 
             var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
 
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var workflow = workflowCreator.CreateWorkflow();
             var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
             var container = containers.Containers[0];

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -4,14 +4,19 @@ namespace DistTestCore.Marketplace
 {
     public class GethContainerRecipe : ContainerRecipeFactory
     {
-        private const string defaultArgs = "--ipcdisable --syncmode full";
+        #if Arm64
+            public const string DockerImage = "emizzle/geth-confenv:latest";
+        #else
+            public const string DockerImage = "thatbenbierens/geth-confenv:onethousand";
+        #endif
 
         public const string HttpPortTag = "http_port";
         public const string DiscoveryPortTag = "disc_port";
+        private const string defaultArgs = "--ipcdisable --syncmode full";
+
         public const string AccountsFilename = "accounts.csv";
 
-        public override string AppName => "geth";
-        public override string Image => "codexstorage/dist-tests-geth:latest";
+        protected override string Image => DockerImage;
 
         protected override void Initialize(StartupConfig startupConfig)
         {

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -1,7 +1,5 @@
 using DistTestCore.Codex;
 using DistTestCore.Helpers;
-using Logging;
-using Newtonsoft.Json;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
 using System.Numerics;
@@ -12,7 +10,7 @@ namespace DistTestCore.Marketplace
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration);
-        StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
+        string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
         void AssertThatBalance(IResolveConstraint constraint, string message = "");
         TestToken GetBalance();
     }
@@ -32,7 +30,7 @@ namespace DistTestCore.Marketplace
             this.codexAccess = codexAccess;
         }
 
-        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        public string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
         {
             var request = new CodexSalesRequestStorageRequest
             {
@@ -59,9 +57,9 @@ namespace DistTestCore.Marketplace
                 throw new InvalidOperationException(response);
             }
 
-            Log($"Storage requested successfully. PurchaseId: '{response}'.");
+            Log($"Storage requested successfully. PurchaseId: {response}");
 
-            return new StoragePurchaseContract(lifecycle.Log, codexAccess, response, duration);
+            return response;
         }
 
         public string MakeStorageAvailable(ByteSize totalSpace, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration)
@@ -123,10 +121,10 @@ namespace DistTestCore.Marketplace
 
     public class MarketplaceUnavailable : IMarketplaceAccess
     {
-        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
         {
             Unavailable();
-            return null!;
+            return string.Empty;
         }
 
         public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration)
@@ -152,92 +150,4 @@ namespace DistTestCore.Marketplace
             throw new InvalidOperationException();
         }
     }
-
-    public class StoragePurchaseContract
-    {
-        private readonly BaseLog log;
-        private readonly CodexAccess codexAccess;
-        private DateTime? contractStartUtc;
-
-        public StoragePurchaseContract(BaseLog log, CodexAccess codexAccess, string purchaseId, TimeSpan contractDuration)
-        {
-            this.log = log;
-            this.codexAccess = codexAccess;
-            PurchaseId = purchaseId;
-            ContractDuration = contractDuration;
-        }
-
-        public string PurchaseId { get; }
-        public TimeSpan ContractDuration { get; }
-
-        public void WaitForStorageContractStarted()
-        {
-            WaitForStorageContractStarted(TimeSpan.FromSeconds(30));
-        }
-
-        public void WaitForStorageContractFinished()
-        {
-            if (!contractStartUtc.HasValue)
-            {
-                WaitForStorageContractStarted();
-            }
-            var gracePeriod = TimeSpan.FromSeconds(10);
-            var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
-            var timeout = (ContractDuration - currentContractTime) + gracePeriod;
-            WaitForStorageContractState(timeout, "finished");
-        }
-
-        /// <summary>
-        /// Wait for contract to start. Max timeout depends on contract filesize. Allows more time for larger files.
-        /// </summary>
-        public void WaitForStorageContractStarted(ByteSize contractFileSize)
-        {
-            var filesizeInMb = contractFileSize.SizeInBytes / (1024 * 1024);
-            var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0);
-
-            WaitForStorageContractStarted(maxWaitTime);
-        }
-
-        public void WaitForStorageContractStarted(TimeSpan timeout)
-        {
-            WaitForStorageContractState(timeout, "started");
-            contractStartUtc = DateTime.UtcNow;
-        }
-
-        private void WaitForStorageContractState(TimeSpan timeout, string desiredState)
-        {
-            var lastState = "";
-            var waitStart = DateTime.UtcNow;
-            
-            log.Log($"Waiting for {Time.FormatDuration(timeout)} for contract '{PurchaseId}' to reach state '{desiredState}'.");
-            while (lastState != desiredState)
-            {
-                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
-                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
-                if (purchaseStatus != null && purchaseStatus.state != lastState)
-                {
-                    lastState = purchaseStatus.state;
-                    log.Debug("Purchase status: " + statusJson);
-                }
-
-                Thread.Sleep(1000);
-
-                if (lastState == "errored")
-                {
-                    Assert.Fail("Contract errored: " + statusJson);
-                }
-
-                if (DateTime.UtcNow - waitStart > timeout)
-                {
-                    Assert.Fail($"Contract did not reach '{desiredState}' within timeout. {statusJson}");
-                }
-            }
-            log.Log($"Contract '{desiredState}'.");
-        }
-
-        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
-        {
-            return codexAccess.GetPurchaseStatus(purchaseId);
-        }
-    }
 }

DistTestCore/Metrics/GrafanaContainerRecipe.cs

@@ -1,25 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore.Metrics
-{
-    public class GrafanaContainerRecipe : ContainerRecipeFactory
-    {
-        public override string AppName => "grafana";
-        public override string Image => "grafana/grafana-oss:10.0.3";
-
-        public const string DefaultAdminUser = "adminium";
-        public const string DefaultAdminPassword = "passwordium";
-
-        protected override void Initialize(StartupConfig startupConfig)
-        {
-            AddExposedPort(3000);
-
-            AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true");
-            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org.");
-            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor");
-
-            AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser);
-            AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword);
-        }
-    }
-}

DistTestCore/Metrics/MetricsMode.cs

@@ -1,9 +0,0 @@
-namespace DistTestCore.Metrics
-{
-    public enum MetricsMode
-    {
-        None,
-        Record,
-        Dashboard
-    }
-}

DistTestCore/Metrics/PrometheusContainerRecipe.cs

@@ -4,8 +4,9 @@ namespace DistTestCore.Metrics
 {
     public class PrometheusContainerRecipe : ContainerRecipeFactory
     {
-        public override string AppName => "prometheus";
-        public override string Image => "codexstorage/dist-tests-prometheus:latest";
+        public const string DockerImage = "thatbenbierens/prometheus-envconf:latest";
+
+        protected override string Image => DockerImage;
 
         protected override void Initialize(StartupConfig startupConfig)
         {

DistTestCore/OnlineCodexNode.cs

@@ -4,7 +4,6 @@ using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
 using Logging;
 using NUnit.Framework;
-using Utils;
 
 namespace DistTestCore
 {
@@ -16,10 +15,9 @@ namespace DistTestCore
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(IOnlineCodexNode node);
-        IDownloadedLog DownloadLog(int? tailLines = null);
+        IDownloadedLog DownloadLog();
         IMetricsAccess Metrics { get; }
         IMarketplaceAccess Marketplace { get; }
-        CodexDebugVersionResponse Version { get; }
         ICodexSetup BringOffline();
     }
 
@@ -36,14 +34,12 @@ namespace DistTestCore
             Group = group;
             Metrics = metricsAccess;
             Marketplace = marketplaceAccess;
-            Version = new CodexDebugVersionResponse();
         }
 
         public CodexAccess CodexAccess { get; }
         public CodexNodeGroup Group { get; }
         public IMetricsAccess Metrics { get; }
         public IMarketplaceAccess Marketplace { get; }
-        public CodexDebugVersionResponse Version { get; private set; }
 
         public string GetName()
         {
@@ -68,7 +64,6 @@ namespace DistTestCore
             using var fileStream = File.OpenRead(file.Filename);
 
             var logMessage = $"Uploading file {file.Describe()}...";
-            Log(logMessage);
             var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
             {
                 return CodexAccess.UploadFile(fileStream);
@@ -77,6 +72,9 @@ namespace DistTestCore
             if (string.IsNullOrEmpty(response)) Assert.Fail("Received empty response.");
             if (response.StartsWith(UploadFailedMessage)) Assert.Fail("Node failed to store block.");
 
+            var logReplacement = $"(CID:{file.Describe()})";
+            Log($"ContentId '{response}' is {logReplacement}");
+            lifecycle.Log.AddStringReplace(response, logReplacement);
             Log($"Uploaded file. Received contentId: '{response}'.");
             return new ContentId(response);
         }
@@ -84,7 +82,6 @@ namespace DistTestCore
         public TestFile? DownloadContent(ContentId contentId, string fileLabel = "")
         {
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
-            Log(logMessage);
             var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel);
             Stopwatch.Measure(lifecycle.Log, logMessage, () => DownloadToFile(contentId.Id, file));
             Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
@@ -103,9 +100,9 @@ namespace DistTestCore
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public IDownloadedLog DownloadLog(int? tailLines = null)
+        public IDownloadedLog DownloadLog()
         {
-            return lifecycle.DownloadLog(CodexAccess.Container, tailLines);
+            return lifecycle.DownloadLog(CodexAccess.Container);
         }
 
         public ICodexSetup BringOffline()
@@ -117,30 +114,19 @@ namespace DistTestCore
             return Group.BringOffline();
         }
 
-        public void EnsureOnlineGetVersionResponse()
-        {
-            var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
-            var nodePeerId = debugInfo.id;
-            var nodeName = CodexAccess.Container.Name;
-
-            if (!debugInfo.codex.IsValid())
-            {
-                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}");
-            }
-
-            lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
-            lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
-            Version = debugInfo.codex;
-        }
-
         private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
         {
             var multiAddress = peerInfo.addrs.First();
             // Todo: Is there a case where First address in list is not the way?
 
+            if (Group == peer.Group)
+            {
+                return multiAddress;
+            }
+
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip);
+            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip);
         }
 
         private void DownloadToFile(string contentId, TestFile file)

DistTestCore/PrometheusStarter.cs

@@ -7,21 +7,23 @@ namespace DistTestCore
 {
     public class PrometheusStarter : BaseStarter
     {
-        public PrometheusStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
+        public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+            : base(lifecycle, workflowCreator)
         {
         }
 
-        public RunningContainers CollectMetricsFor(RunningContainers[] containers)
+        public RunningContainers CollectMetricsFor(RunningContainers containers)
         {
             LogStart($"Starting metrics server for {containers.Describe()}");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers())));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers)));
 
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var workflow = workflowCreator.CreateWorkflow();
             var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 
+            LogEnd("Metrics server started.");
+
             return runningContainers;
         }
 
@@ -29,7 +31,7 @@ namespace DistTestCore
         {
             var config = "";
             config += "global:\n";
-            config += "  scrape_interval: 10s\n";
+            config += "  scrape_interval: 30s\n";
             config += "  scrape_timeout: 10s\n";
             config += "\n";
             config += "scrape_configs:\n";

DistTestCore/TestLifecycle.cs

@@ -1,7 +1,4 @@
-using DistTestCore.Codex;
-using DistTestCore.Logs;
-using DistTestCore.Marketplace;
-using DistTestCore.Metrics;
+using DistTestCore.Logs;
 using KubernetesWorkflow;
 using Logging;
 using Utils;
@@ -10,38 +7,33 @@ namespace DistTestCore
 {
     public class TestLifecycle
     {
-        private readonly DateTime testStart;
+        private DateTime testStart = DateTime.MinValue;
 
-        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testsType, string testNamespace)
+        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet)
+            : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)))
+        {
+        }
+
+        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
         {
             Log = log;
             Configuration = configuration;
             TimeSet = timeSet;
 
-            var podLabels = new PodLabels(testsType, GetApplicationIds());
-            WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), podLabels, testNamespace);
-
             FileManager = new FileManager(Log, configuration);
-            CodexStarter = new CodexStarter(this);
-            PrometheusStarter = new PrometheusStarter(this);
-            GrafanaStarter = new GrafanaStarter(this);
-            GethStarter = new GethStarter(this);
+            CodexStarter = new CodexStarter(this, workflowCreator);
+            PrometheusStarter = new PrometheusStarter(this, workflowCreator);
+            GethStarter = new GethStarter(this, workflowCreator);
             testStart = DateTime.UtcNow;
-            CodexVersion = null;
-
-            Log.WriteLogTag();
         }
 
         public BaseLog Log { get; }
         public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
-        public WorkflowCreator WorkflowCreator { get; }
         public FileManager FileManager { get; }
         public CodexStarter CodexStarter { get; }
         public PrometheusStarter PrometheusStarter { get; }
-        public GrafanaStarter GrafanaStarter { get; }
         public GethStarter GethStarter { get; }
-        public CodexDebugVersionResponse? CodexVersion { get; private set; }
 
         public void DeleteAllResources()
         {
@@ -49,14 +41,14 @@ namespace DistTestCore
             FileManager.DeleteAllTestFiles();
         }
 
-        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
+        public IDownloadedLog DownloadLog(RunningContainer container)
         {
             var subFile = Log.CreateSubfile();
             var description = container.Name;
             var handler = new LogDownloadHandler(container, description, subFile);
 
             Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-            CodexStarter.DownloadLog(container, handler, tailLines);
+            CodexStarter.DownloadLog(container, handler);
 
             return new DownloadedLog(subFile, description);
         }
@@ -66,29 +58,5 @@ namespace DistTestCore
             var testDuration = DateTime.UtcNow - testStart;
             return Time.FormatDuration(testDuration);
         }
-
-        public void SetCodexVersion(CodexDebugVersionResponse version)
-        {
-            if (CodexVersion == null) CodexVersion = version;
-        }
-
-        public ApplicationIds GetApplicationIds()
-        {
-            return new ApplicationIds(
-                codexId: GetCodexId(),
-                gethId: new GethContainerRecipe().Image,
-                prometheusId: new PrometheusContainerRecipe().Image,
-                codexContractsId: new CodexContractsContainerRecipe().Image,
-                grafanaId: new GrafanaContainerRecipe().Image
-            );
-        }
-
-        private string GetCodexId()
-        {
-            var v = CodexVersion;
-            if (v == null) return new CodexContainerRecipe().Image;
-            if (v.version != "untagged build") return v.version;
-            return v.revision;
-        }
     }
 }

DistTestCore/Timing.cs

@@ -21,7 +21,7 @@ namespace DistTestCore
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromMinutes(5);
+            return TimeSpan.FromSeconds(10);
         }
 
         public TimeSpan HttpCallRetryTime()
@@ -36,12 +36,12 @@ namespace DistTestCore
 
         public TimeSpan WaitForK8sServiceDelay()
         {
-            return TimeSpan.FromSeconds(10);
+            return TimeSpan.FromSeconds(1);
         }
 
         public TimeSpan K8sOperationTimeout()
         {
-            return TimeSpan.FromMinutes(30);
+            return TimeSpan.FromMinutes(1);
         }
 
         public TimeSpan WaitForMetricTimeout()

DistTestCore/cs-codex-dist-tests.code-workspace

@@ -0,0 +1,11 @@
+{
+	"folders": [
+		{
+			"path": ".."
+		},
+		{
+			"path": "../../../CodexTestLogs"
+		}
+	],
+	"settings": {}
+}

KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -27,8 +27,7 @@
             return recipe;
         }
 
-        public abstract string AppName { get; }
-        public abstract string Image { get; }
+        protected abstract string Image { get; }
         protected int ContainerNumber { get; private set; } = 0;
         protected int Index { get; private set; } = 0;
         protected abstract void Initialize(StartupConfig config);
@@ -40,13 +39,6 @@
             return p;
         }
 
-        protected Port AddExposedPort(int number, string tag = "")
-        {
-            var p = factory.CreatePort(number, tag);
-            exposedPorts.Add(p);
-            return p;
-        }
-
         protected Port AddInternalPort(string tag = "")
         {
             var p = factory.CreatePort(tag);

KubernetesWorkflow/CrashWatcher.cs

@@ -1,95 +0,0 @@
-using k8s;
-using Logging;
-
-namespace KubernetesWorkflow
-{
-    public class CrashWatcher
-    {
-        private readonly BaseLog log;
-        private readonly KubernetesClientConfiguration config;
-        private readonly string k8sNamespace;
-        private readonly RunningContainer container;
-        private ILogHandler? logHandler;
-        private CancellationTokenSource cts;
-        private Task? worker;
-        private Exception? workerException;
-
-        public CrashWatcher(BaseLog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container)
-        {
-            this.log = log;
-            this.config = config;
-            this.k8sNamespace = k8sNamespace;
-            this.container = container;
-            cts = new CancellationTokenSource();
-        }
-
-        public void Start(ILogHandler logHandler)
-        {
-            if (worker != null) throw new InvalidOperationException();
-
-            this.logHandler = logHandler;
-            cts = new CancellationTokenSource();
-            worker = Task.Run(Worker);
-        }
-
-        public void Stop()
-        {
-            if (worker == null) throw new InvalidOperationException();
-
-            cts.Cancel();
-            worker.Wait();
-            worker = null;
-
-            if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
-        }
-
-        public bool HasContainerCrashed()
-        {
-            using var client = new Kubernetes(config);
-            return HasContainerBeenRestarted(client, container.Pod.PodInfo.Name);
-        }
-
-        private void Worker()
-        {
-            try
-            {
-                MonitorContainer(cts.Token);
-            }
-            catch (Exception ex)
-            {
-                workerException = ex;
-            }
-        }
-
-        private void MonitorContainer(CancellationToken token)
-        {
-            using var client = new Kubernetes(config);
-            while (!token.IsCancellationRequested)
-            {
-                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(1));
-
-                var pod = container.Pod;
-                var recipe = container.Recipe;
-                var podName = pod.PodInfo.Name;
-                if (HasContainerBeenRestarted(client, podName))
-                {
-                    DownloadCrashedContainerLogs(client, podName, recipe);
-                    return;
-                }
-            }
-        }
-
-        private bool HasContainerBeenRestarted(Kubernetes client, string podName)
-        {
-            var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
-            return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
-        }
-
-        private void DownloadCrashedContainerLogs(Kubernetes client, string podName, ContainerRecipe recipe)
-        {
-            log.Log("Pod crash detected for " + container.Name);
-            using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipe.Name, previous: true);
-            logHandler!.Log(stream);
-        }
-    }
-}

KubernetesWorkflow/K8sController.cs

@@ -11,16 +11,14 @@ namespace KubernetesWorkflow
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownPods;
         private readonly WorkflowNumberSource workflowNumberSource;
-        private readonly PodLabels podLabels;
         private readonly K8sClient client;
 
-        public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace, PodLabels podLabels)
+        public K8sController(BaseLog log, K8sCluster cluster, KnownK8sPods knownPods, WorkflowNumberSource workflowNumberSource, string testNamespace)
         {
             this.log = log;
             this.cluster = cluster;
             this.knownPods = knownPods;
             this.workflowNumberSource = workflowNumberSource;
-            this.podLabels = podLabels;
             client = new K8sClient(cluster.GetK8sClientConfig());
 
             K8sTestNamespace = cluster.Configuration.K8sNamespacePrefix + testNamespace;
@@ -53,10 +51,10 @@ namespace KubernetesWorkflow
             WaitUntilPodOffline(pod.PodInfo.Name);
         }
 
-        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler, int? tailLines)
+        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
         {
             log.Debug();
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, tailLines: tailLines));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name));
             logHandler.Log(stream);
         }
 
@@ -364,7 +362,7 @@ namespace KubernetesWorkflow
 
         private IDictionary<string, string> GetSelector()
         {
-            return podLabels.GetLabels();
+            return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
         }
 
         private IDictionary<string, string> GetRunnerNamespaceSelector()
@@ -604,11 +602,6 @@ namespace KubernetesWorkflow
 
         #endregion
 
-        public CrashWatcher CreateCrashWatcher(RunningContainer container)
-        {
-            return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sTestNamespace, container);
-        }
-
         private PodInfo FetchNewPod()
         {
             var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;

KubernetesWorkflow/PodLabels.cs

@@ -1,61 +0,0 @@
-using Logging;
-
-namespace KubernetesWorkflow
-{
-    public class PodLabels
-    {
-        private readonly Dictionary<string, string> labels = new Dictionary<string, string>();
-
-        private PodLabels(PodLabels source)
-        {
-            labels = source.labels.ToDictionary(p => p.Key, p => p.Value);
-        }
-
-        public PodLabels(string testsType, ApplicationIds applicationIds)
-        {
-            Add("tests-type", testsType);
-            Add("runid", NameUtils.GetRunId());
-            Add("testid", NameUtils.GetTestId());
-            Add("category", NameUtils.GetCategoryName());
-            Add("fixturename", NameUtils.GetRawFixtureName());
-            Add("testname", NameUtils.GetTestMethodName());
-
-            if (applicationIds == null) return;
-            Add("codexid", applicationIds.CodexId);
-            Add("gethid", applicationIds.GethId);
-            Add("prometheusid", applicationIds.PrometheusId);
-            Add("codexcontractsid", applicationIds.CodexContractsId);
-            Add("grafanaid", applicationIds.GrafanaId);
-        }
-
-        public PodLabels GetLabelsForAppName(string appName)
-        {
-            var pl = new PodLabels(this);
-            pl.Add("app", appName);
-            return pl;
-        }
-
-        private void Add(string key, string value)
-        {
-            labels.Add(key, Format(value));
-        }
-
-        private static string Format(string s)
-        {
-            var result = s.ToLowerInvariant()
-                .Replace(":", "-")
-                .Replace("/", "-")
-                .Replace("\\", "-")
-                .Replace("[", "-")
-                .Replace("]", "-")
-                .Replace(",", "-");
-
-            return result.Trim('-');
-        }
-
-        internal Dictionary<string, string> GetLabels()
-        {
-            return labels;
-        }
-    }
-}

KubernetesWorkflow/RecipeComponentFactory.cs

@@ -7,11 +7,6 @@ namespace KubernetesWorkflow
     {
         private NumberSource portNumberSource = new NumberSource(8080);
 
-        public Port CreatePort(int number, string tag)
-        {
-            return new Port(number, tag);
-        }
-
         public Port CreatePort(string tag)
         {
             return new Port(portNumberSource.GetNextNumber(), tag);

KubernetesWorkflow/RunningContainers.cs

@@ -1,5 +1,4 @@
-using Newtonsoft.Json;
-using Utils;
+using Utils;
 
 namespace KubernetesWorkflow
 {
@@ -40,21 +39,5 @@ namespace KubernetesWorkflow
         public Port[] ServicePorts { get; }
         public Address ClusterExternalAddress { get; }
         public Address ClusterInternalAddress { get; }
-
-        [JsonIgnore]
-        public CrashWatcher? CrashWatcher { get; set; }
-    }
-
-    public static class RunningContainersExtensions
-    {
-        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
-        {
-            return runningContainers.SelectMany(c => c.Containers).ToArray();
-        }
-
-        public static string Describe(this RunningContainers[] runningContainers)
-        {
-            return string.Join(",", runningContainers.Select(c => c.Describe()));
-        }
     }
 }

KubernetesWorkflow/StartupWorkflow.cs

@@ -10,23 +10,19 @@ namespace KubernetesWorkflow
         private readonly K8sCluster cluster;
         private readonly KnownK8sPods knownK8SPods;
         private readonly string testNamespace;
-        private readonly PodLabels podLabels;
         private readonly RecipeComponentFactory componentFactory = new RecipeComponentFactory();
 
-        internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace, PodLabels podLabels)
+        internal StartupWorkflow(BaseLog log, WorkflowNumberSource numberSource, K8sCluster cluster, KnownK8sPods knownK8SPods, string testNamespace)
         {
             this.log = log;
             this.numberSource = numberSource;
             this.cluster = cluster;
             this.knownK8SPods = knownK8SPods;
             this.testNamespace = testNamespace;
-            this.podLabels = podLabels;
         }
 
         public RunningContainers Start(int numberOfContainers, Location location, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
         {
-            var pl = podLabels.GetLabelsForAppName(recipeFactory.AppName);
-
             return K8s(controller =>
             {
                 var recipes = CreateRecipes(numberOfContainers, recipeFactory, startupConfig);
@@ -34,12 +30,7 @@ namespace KubernetesWorkflow
                 var runningPod = controller.BringOnline(recipes, location);
 
                 return new RunningContainers(startupConfig, runningPod, CreateContainers(runningPod, recipes, startupConfig));
-            }, pl);
-        }
-
-        public CrashWatcher CreateCrashWatcher(RunningContainer container)
-        {
-            return K8s(controller => controller.CreateCrashWatcher(container));
+            });
         }
 
         public void Stop(RunningContainers runningContainers)
@@ -50,11 +41,11 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
+        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler)
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler, tailLines);
+                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler);
             });
         }
 
@@ -156,22 +147,14 @@ namespace KubernetesWorkflow
 
         private void K8s(Action<K8sController> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
             action(controller);
             controller.Dispose();
         }
 
         private T K8s<T>(Func<K8sController, T> action)
         {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, podLabels);
-            var result = action(controller);
-            controller.Dispose();
-            return result;
-        }
-
-        private T K8s<T>(Func<K8sController, T> action, PodLabels labels)
-        {
-            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace, labels);
+            var controller = new K8sController(log, cluster, knownK8SPods, numberSource, testNamespace);
             var result = action(controller);
             controller.Dispose();
             return result;

KubernetesWorkflow/WorkflowCreator.cs

@@ -10,15 +10,18 @@ namespace KubernetesWorkflow
         private readonly KnownK8sPods knownPods = new KnownK8sPods();
         private readonly K8sCluster cluster;
         private readonly BaseLog log;
-        private readonly PodLabels podLabels;
         private readonly string testNamespace;
 
-        public WorkflowCreator(BaseLog log, Configuration configuration, PodLabels podLabels, string testNamespace)
+        public WorkflowCreator(BaseLog log, Configuration configuration)
+            : this(log, configuration, Guid.NewGuid().ToString().ToLowerInvariant())
+        {
+        }
+
+        public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespacePostfix)
         {
             cluster = new K8sCluster(configuration);
             this.log = log;
-            this.podLabels = podLabels;
-            this.testNamespace = testNamespace.ToLowerInvariant();
+            testNamespace = testNamespacePostfix;
         }
 
         public StartupWorkflow CreateWorkflow()
@@ -26,7 +29,7 @@ namespace KubernetesWorkflow
             var workflowNumberSource = new WorkflowNumberSource(numberSource.GetNextNumber(),
                                                                     containerNumberSource);
 
-            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace, podLabels);
+            return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);
         }
     }
 }

Logging/ApplicationIds.cs

@@ -1,20 +0,0 @@
-namespace Logging
-{
-    public class ApplicationIds
-    {
-        public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId, string grafanaId)
-        {
-            CodexId = codexId;
-            GethId = gethId;
-            PrometheusId = prometheusId;
-            CodexContractsId = codexContractsId;
-            GrafanaId = grafanaId;
-        }
-
-        public string CodexId { get; }
-        public string GethId { get; }
-        public string PrometheusId { get; }
-        public string CodexContractsId { get; }
-        public string GrafanaId { get; }
-    }
-}

Logging/BaseLog.cs

@@ -73,14 +73,6 @@ namespace Logging
             return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
         }
 
-        public void WriteLogTag()
-        {
-            var runId = NameUtils.GetRunId();
-            var category = NameUtils.GetCategoryName();
-            var name = NameUtils.GetTestMethodName();
-            LogFile.WriteRaw($"{runId} {category} {name}");
-        }
-
         private string ApplyReplacements(string str)
         {
             foreach (var replacement in replacements)

Logging/FixtureLog.cs

@@ -1,14 +1,20 @@
-namespace Logging
+using NUnit.Framework;
+
+namespace Logging
 {
     public class FixtureLog : BaseLog
     {
+        private readonly DateTime start;
         private readonly string fullName;
         private readonly LogConfig config;
 
-        public FixtureLog(LogConfig config, DateTime start, string name = "")
+        public FixtureLog(LogConfig config, string name = "")
             : base(config.DebugEnabled)
         {
-            fullName = NameUtils.GetFixtureFullName(config, start, name);
+            start = DateTime.UtcNow;
+            var folder = DetermineFolder(config);
+            var fixtureName = GetFixtureName(name);
+            fullName = Path.Combine(folder, fixtureName);
             this.config = config;
         }
 
@@ -26,5 +32,28 @@
         {
             return fullName;
         }
+
+        private string DetermineFolder(LogConfig config)
+        {
+            return Path.Join(
+               config.LogRoot,
+               $"{start.Year}-{Pad(start.Month)}",
+               Pad(start.Day));
+        }
+
+        private string GetFixtureName(string name)
+        {
+            var test = TestContext.CurrentContext.Test;
+            var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
+            if (!string.IsNullOrEmpty(name)) className = name;
+
+            return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
+        }
+
+        private static string Pad(int n)
+        {
+            return n.ToString().PadLeft(2, '0');
+        }
+
     }
 }

Logging/LogFile.cs

@@ -49,7 +49,7 @@
 
         private static string GetTimestamp()
         {
-            return $"[{DateTime.UtcNow.ToString("o")}]";
+            return $"[{DateTime.UtcNow.ToString("u")}]";
         }
 
         private void EnsurePathExists(string filename)

Logging/NameUtils.cs

@@ -1,85 +0,0 @@
-using NUnit.Framework;
-
-namespace Logging
-{
-    public static class NameUtils
-    {
-        public static string GetTestMethodName(string name = "")
-        {
-            if (!string.IsNullOrEmpty(name)) return name;
-            var test = TestContext.CurrentContext.Test;
-            var args = FormatArguments(test);
-            return ReplaceInvalidCharacters($"{test.MethodName}{args}");
-        }
-
-        public static string GetFixtureFullName(LogConfig config, DateTime start, string name)
-        {
-            var folder = DetermineFolder(config, start);
-            var fixtureName = GetFixtureName(name, start);
-            return Path.Combine(folder, fixtureName);
-        }
-
-        public static string GetRawFixtureName()
-        {
-            var test = TestContext.CurrentContext.Test;
-            if (test.ClassName!.Contains("AdhocContext")) return "none";
-            var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
-            return className.Replace('.', '-');
-        }
-
-        public static string GetCategoryName()
-        {
-            var test = TestContext.CurrentContext.Test;
-            if (test.ClassName!.Contains("AdhocContext")) return "none";
-            return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.'));
-        }
-
-        public static string GetTestId()
-        {
-            return GetEnvVar("TESTID");
-        }
-
-        public static string GetRunId()
-        {
-            return GetEnvVar("RUNID");
-        }
-
-        private static string GetEnvVar(string name)
-        {
-            var v = Environment.GetEnvironmentVariable(name);
-            if (string.IsNullOrEmpty(v)) return $"EnvVar-{name}-NotSet";
-            return v;
-        }
-
-        private static string FormatArguments(TestContext.TestAdapter test)
-        {
-            if (test.Arguments == null || !test.Arguments.Any()) return "";
-            return $"[{string.Join(',', test.Arguments)}]";
-        }
-
-        private static string ReplaceInvalidCharacters(string name)
-        {
-            return name.Replace(":", "_");
-        }
-
-        private static string DetermineFolder(LogConfig config, DateTime start)
-        {
-            return Path.Join(
-               config.LogRoot,
-               $"{start.Year}-{Pad(start.Month)}",
-               Pad(start.Day));
-        }
-
-        private static string GetFixtureName(string name, DateTime start)
-        {
-            var className = GetRawFixtureName();
-            if (!string.IsNullOrEmpty(name)) className = name;
-            return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
-        }
-
-        private static string Pad(int n)
-        {
-            return n.ToString().PadLeft(2, '0');
-        }
-    }
-}

Logging/StatusLog.cs

@@ -1,69 +0,0 @@
-using Newtonsoft.Json;
-
-namespace Logging
-{
-    public class StatusLog
-    {
-        private readonly object fileLock = new object();
-        private readonly string fullName;
-        private readonly string fixtureName;
-
-        public StatusLog(LogConfig config, DateTime start, string name = "")
-        {
-            fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log";
-            fixtureName = NameUtils.GetRawFixtureName();
-        }
-
-        public void ConcludeTest(string resultStatus, string testDuration, ApplicationIds applicationIds)
-        {
-            Write(new StatusLogJson
-            {
-                @timestamp = DateTime.UtcNow.ToString("o"),
-                runid = NameUtils.GetRunId(),
-                status = resultStatus,
-                testid = NameUtils.GetTestId(),
-                codexid = applicationIds.CodexId,
-                gethid = applicationIds.GethId,
-                prometheusid = applicationIds.PrometheusId,
-                codexcontractsid = applicationIds.CodexContractsId,
-                grafanaid = applicationIds.GrafanaId,
-                category = NameUtils.GetCategoryName(),
-                fixturename = fixtureName,
-                testname = NameUtils.GetTestMethodName(),
-                testduration = testDuration
-            });
-        }
-
-        private void Write(StatusLogJson json)
-        {
-            try
-            {
-                lock (fileLock)
-                {
-                    File.AppendAllLines(fullName, new[] { JsonConvert.SerializeObject(json) });
-                }
-            }
-            catch (Exception ex)
-            {
-                Console.WriteLine("Unable to write to status log: " + ex);
-            }
-        }
-    }
-
-    public class StatusLogJson
-    {
-        public string @timestamp { get; set; } = string.Empty;
-        public string runid { get; set; } = string.Empty;
-        public string status { get; set; } = string.Empty;
-        public string testid { get; set; } = string.Empty; 
-        public string codexid { get; set; } = string.Empty;
-        public string gethid { get; set; } = string.Empty;
-        public string prometheusid { get; set; } = string.Empty;
-        public string codexcontractsid { get; set; } = string.Empty;
-        public string grafanaid { get; set; } = string.Empty;
-        public string category { get; set; } = string.Empty;
-        public string fixturename { get; set; } = string.Empty;
-        public string testname { get; set; } = string.Empty;
-        public string testduration { get; set;} = string.Empty;
-    }
-}

Logging/TestLog.cs

@@ -10,7 +10,7 @@ namespace Logging
         public TestLog(string folder, bool debug, string name = "")
             : base(debug)
         {
-            methodName = NameUtils.GetTestMethodName(name);
+            methodName = GetMethodName(name);
             fullName = Path.Combine(folder, methodName);
 
             Log($"*** Begin: {methodName}");
@@ -37,5 +37,24 @@ namespace Logging
         {
             return fullName;
         }
+
+        private string GetMethodName(string name)
+        {
+            if (!string.IsNullOrEmpty(name)) return name;
+            var test = TestContext.CurrentContext.Test;
+            var args = FormatArguments(test);
+            return ReplaceInvalidCharacters($"{test.MethodName}{args}");
+        }
+
+        private static string FormatArguments(TestContext.TestAdapter test)
+        {
+            if (test.Arguments == null || !test.Arguments.Any()) return "";
+            return $"[{string.Join(',', test.Arguments)}]";
+        }
+
+        private static string ReplaceInvalidCharacters(string name)
+        {
+            return name.Replace(":", "_");
+        }
     }
 }

LongTests/BasicTests/DownloadTests.cs

@@ -1,5 +1,7 @@
 using DistTestCore;
 using NUnit.Framework;
+using k8s;
+using k8s.Models;
 
 namespace TestsLong.BasicTests
 {
@@ -23,7 +25,8 @@ namespace TestsLong.BasicTests
             var testFile = GenerateTestFile(filesizeMb.MB());
             var contentId = host.UploadFile(testFile);
             var list = new List<Task<TestFile?>>();
-
+            //sleep for 1 minute
+            Thread.Sleep(1200000);
             foreach (var node in group)
             {
                 list.Add(Task.Run(() => { return node.DownloadContent(contentId); }));

LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs

@@ -1,4 +1,5 @@
-using DistTestCore;
+using DistTestCore.Helpers;
+using DistTestCore;
 using NUnit.Framework;
 
 namespace TestsLong.DownloadConnectivityTests
@@ -15,7 +16,7 @@ namespace TestsLong.DownloadConnectivityTests
         {
             for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
 
-            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
+            PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
         }
     }
 }

README.md

@@ -3,6 +3,7 @@
 
 Using a common dotnet unit-test framework and a few other libraries, this project allows you to write tests that use multiple Codex node instances in various configurations to test the distributed system in a controlled, reproducible environment.
 
+
 Nim-Codex: https://github.com/codex-storage/nim-codex  
 Dotnet: v6.0  
 Kubernetes: v1.25.4  
@@ -22,7 +23,7 @@ Test executing can be configured using the following environment variables.
 |----------------|------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|
 | KUBECONFIG     | Optional path (abs or rel) to kubeconfig YAML file. When null, uses system default (docker-desktop) kubeconfig if available.                         | (null)              |
 | LOGPATH        | Path (abs or rel) where log files will be saved.                                                                                                     | "CodexTestLogs"     |
-| LOGDEBUG       | When "true", enables additional test-runner debug log output.                                                                                        | "false"             |
+| LOGDEBUG       | When "true", enables additional test-runner debug log output.                                                                                                    | "false"             |
 | DATAFILEPATH   | Path (abs or rel) where temporary test data files will be saved.                                                                                     | "TestDataFiles"     |
 | LOGLEVEL       | Codex log-level. (case-insensitive)                                                                                                                  | "Trace"             |
 | RUNNERLOCATION | Use "ExternalToCluster" when test app is running outside of the k8s cluster. Use "InternalToCluster" when tests are run from inside a pod/container. | "ExternalToCluster" |
@@ -31,10 +32,10 @@ Test executing can be configured using the following environment variables.
 Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs.
 
 ## How to contribute tests
-An important goal of the test infra is to provide a simple, accessible way for developers to write their tests. If you want to contribute tests for Codex, please follow the steps [HERE](/CONTRIBUTINGTESTS.md).
+An important goal of the test infra is to provide a simple, accessible way for developers to write their tests. If you want to contribute tests for Codex, please follow the steps [HERE](/CONTRIBUTINGTESTS.MD).
 
 ## Run the tests on your machine
-Creating tests is much easier when you can debug them on your local system. This is possible, but requires some set-up. If you want to be able to run the tests on your local system, follow the steps [HERE](/docs/LOCALSETUP.md). Please note that tests which require explicit node locations cannot be executed locally. (Well, you could comment out the location statements and then it would probably work. But that might impact the validity/usefulness of the test.)
+Creating tests is much easier when you can debug them on your local system. This is possible, but requires some set-up. If you want to be able to run the tests on your local system, follow the steps [HERE](/LOCALSETUP.MD). Please note that tests which require explicit node locations cannot be executed locally. (Well, you could comment out the location statements and then it would probably work. But that might impact the validity/usefulness of the test.)
 
 ## Missing functionality
 Surely the test-infra doesn't do everything we'll need it to do. If you're running into a limitation and would like to request a new feature for the test-infra, please create an issue.

Tests/BasicTests/ContinuousSubstitute.cs

@@ -1,194 +0,0 @@
-using DistTestCore;
-using Logging;
-using NUnit.Framework;
-using Utils;
-
-namespace Tests.BasicTests
-{
-    [TestFixture]
-    public class ContinuousSubstitute : AutoBootstrapDistTest
-    {
-        [Test]
-        [UseLongTimeouts]
-        public void ContinuousTestSubstitute()
-        {
-            var group = SetupCodexNodes(5, o => o
-                    .EnableMetrics()
-                    .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
-                    .WithBlockTTL(TimeSpan.FromMinutes(2))
-                    .WithStorageQuota(3.GB()));
-
-            var nodes = group.Cast<OnlineCodexNode>().ToArray();
-
-            foreach (var node in nodes)
-            {
-                node.Marketplace.MakeStorageAvailable(
-                size: 1.GB(),
-                minPricePerBytePerSecond: 1.TestTokens(),
-                maxCollateral: 1024.TestTokens(),
-                maxDuration: TimeSpan.FromMinutes(5));
-            }
-
-            var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
-            while (DateTime.UtcNow < endTime)
-            {
-                var allNodes = nodes.ToList();
-                var primary = allNodes.PickOneRandom();
-                var secondary = allNodes.PickOneRandom();
-
-                Log("Run Test");
-                PerformTest(primary, secondary);
-
-                Thread.Sleep(TimeSpan.FromSeconds(5));
-            }
-        }
-
-        private ByteSize fileSize = 80.MB();
-
-        private void PerformTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
-        {
-            ScopedTestFiles(() =>
-            {
-                var testFile = GenerateTestFile(fileSize);
-
-                var contentId = primary.UploadFile(testFile);
-
-                var downloadedFile = secondary.DownloadContent(contentId);
-
-                testFile.AssertIsEqual(downloadedFile);
-            });
-        }
-        
-        [Test]
-        public void HoldMyBeerTest()
-        {
-            var blockExpirationTime = TimeSpan.FromMinutes(3);
-            var group = SetupCodexNodes(3, o => o
-                    .EnableMetrics()
-                    .WithBlockTTL(blockExpirationTime)
-                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
-                    .WithBlockMaintenanceNumber(10000)
-                    .WithStorageQuota(2000.MB()));
-
-            var nodes = group.Cast<OnlineCodexNode>().ToArray();
-
-            var endTime = DateTime.UtcNow + TimeSpan.FromHours(24);
-
-            var filesize = 80.MB();
-            double codexDefaultBlockSize = 31 * 64 * 33;
-            var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
-            var sizeInBytes = filesize.SizeInBytes;
-            Assert.That(numberOfBlocks, Is.EqualTo(1282));
-
-            var startTime = DateTime.UtcNow;
-            var successfulUploads = 0;
-            var successfulDownloads = 0;
-
-            while (DateTime.UtcNow < endTime)
-            {
-                foreach (var node in nodes)
-                {
-                    try
-                    {
-                        Thread.Sleep(TimeSpan.FromSeconds(5));
-
-                        ScopedTestFiles(() =>
-                        {
-                            var uploadStartTime = DateTime.UtcNow;
-                            var file = GenerateTestFile(filesize);
-                            var cid = node.UploadFile(file);
-
-                            var cidTag = cid.Id.Substring(cid.Id.Length - 6);
-                            Measure("upload-log-asserts", () =>
-                            {
-                                var uploadLog = node.DownloadLog(tailLines: 50000);
-
-                                var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
-                                uploadLog.DeleteFile();
-
-                                var storeLine = GetLineForCidTag(storeLines, cidTag);
-                                AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
-                            });
-                            successfulUploads++;
-
-                            var uploadTimeTaken = DateTime.UtcNow - uploadStartTime;
-                            if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10)))
-                            {
-                                Assert.Fail("Upload took too long. Blocks already expired.");
-                            }
-
-                            var dl = node.DownloadContent(cid);
-                            file.AssertIsEqual(dl);
-
-                            Measure("download-log-asserts", () =>
-                            {
-                                var downloadLog = node.DownloadLog(tailLines: 50000);
-
-                                var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
-                                downloadLog.DeleteFile();
-
-                                var sentLine = GetLineForCidTag(sentLines, cidTag);
-                                AssertSentLineContains(sentLine, sizeInBytes);
-                            });
-                            successfulDownloads++;
-                        });
-                    }
-                    catch
-                    {
-                        var testDuration = DateTime.UtcNow - startTime;
-                        Log("Test failed. Delaying shut-down by 30 seconds to collect metrics.");
-                        Log($"Test failed after {Time.FormatDuration(testDuration)} and {successfulUploads} successful uploads and {successfulDownloads} successful downloads");
-                        Thread.Sleep(TimeSpan.FromSeconds(30));
-                        throw;
-                    }
-                }
-
-                Thread.Sleep(TimeSpan.FromSeconds(5));
-            }
-        }
-
-        private void AssertSentLineContains(string sentLine, long sizeInBytes)
-        {
-            var tag = "bytes=";
-            var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length);
-            var bytes = Convert.ToInt64(token);
-            Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'");
-        }
-
-        private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes)
-        {
-            var tokens = storeLine.Split(" ");
-
-            var blocksToken = GetToken(tokens, "blocks=");
-            var sizeToken = GetToken(tokens, "size=");
-            if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine);
-            if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine);
-
-            var blocks = Convert.ToInt64(blocksToken);
-            var size = Convert.ToInt64(sizeToken?.Replace("'NByte", ""));
-
-            var lineLog = $" Line: '{storeLine}'";
-            Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog);
-            Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog);
-        }
-
-        private string GetLineForCidTag(string[] lines, string cidTag)
-        {
-            var result = lines.SingleOrDefault(l => l.Contains(cidTag));
-            if (result == null)
-            {
-                Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'");
-                throw new Exception();
-            }
-
-            return result;
-        }
-
-        private string? GetToken(string[] tokens, string tag)
-        {
-            var token = tokens.SingleOrDefault(t => t.StartsWith(tag));
-            if (token == null) return null;
-            return token.Substring(tag.Length);
-        }
-    }
-}

Tests/BasicTests/ExampleTests.cs

@@ -1,5 +1,6 @@
 using DistTestCore;
 using NUnit.Framework;
+using Utils;
 
 namespace Tests.BasicTests
 {
@@ -43,7 +44,6 @@ namespace Tests.BasicTests
         {
             var sellerInitialBalance = 234.TestTokens();
             var buyerInitialBalance = 1000.TestTokens();
-            var fileSize = 10.MB();
 
             var seller = SetupCodexNode(s => s
                             .WithStorageQuota(11.GB())
@@ -56,27 +56,25 @@ namespace Tests.BasicTests
                 maxCollateral: 20.TestTokens(),
                 maxDuration: TimeSpan.FromMinutes(3));
 
-            var testFile = GenerateTestFile(fileSize);
+            var testFile = GenerateTestFile(10.MB());
 
             var buyer = SetupCodexNode(s => s
                 .WithBootstrapNode(seller)
                 .EnableMarketplace(buyerInitialBalance));
-
-            buyer.Marketplace.AssertThatBalance(Is.EqualTo(buyerInitialBalance));
             
             var contentId = buyer.UploadFile(testFile);
-            var purchaseContract = buyer.Marketplace.RequestStorage(contentId,
+            buyer.Marketplace.RequestStorage(contentId,
                 pricePerSlotPerSecond: 2.TestTokens(),
                 requiredCollateral: 10.TestTokens(),
                 minRequiredNumberOfNodes: 1,
                 proofProbability: 5,
                 duration: TimeSpan.FromMinutes(1));
 
-            purchaseContract.WaitForStorageContractStarted(fileSize);
+            Time.Sleep(TimeSpan.FromSeconds(10));
 
             seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
 
-            purchaseContract.WaitForStorageContractFinished();
+            Time.Sleep(TimeSpan.FromMinutes(1));
 
             seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
             buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");

Tests/BasicTests/MixedTests.cs

@@ -0,0 +1,49 @@
+using DistTestCore;
+using NUnit.Framework;
+
+namespace Tests.ParallelTests
+{
+    [TestFixture]
+    public class MixedTests : DistTest
+    {
+        [TestCase(1, 10)]
+        [UseLongTimeouts]
+        public void ParallelMixed(int numberOfNodes, int filesizeMb)
+        {
+            // initialize the nodes
+            var group = SetupCodexNodes(numberOfNodes);
+            var host = SetupCodexNode();
+
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+            // Upload single file for the download nodes
+            var testfile = GenerateTestFile(filesizeMb.MB());
+            var contentId = host.UploadFile(testfile);
+
+            var testfiles = new List<TestFile>();
+            var contentIds = new List<Task<ContentId>>();
+
+            // Starts uploads for the upload nodes
+            for (int i = 0; i < group.Count(); i++)
+            {
+                testfiles.Add(GenerateTestFile(filesizeMb.MB()));
+                var n = i;
+                contentIds.Add(Task.Run(() => { return host.UploadFile(testfiles[n]); }));
+            }
+            // Starts downloads for the download nodes
+            var downloads = new List<Task<TestFile?>>();
+            for (int i = 0; i < group.Count(); i++)
+            {
+                var n = i;
+                downloads.Add(Task.Run(() => { return group[n].DownloadContent(contentId); }));
+            }
+            Task.WaitAll(downloads.ToArray());
+            for (int i = 0; i < group.Count(); i++)
+            {
+                testfiles[i].AssertIsEqual(downloads[i].Result);
+            }
+        }
+    }
+}

Tests/BasicTests/TwoClientTests.cs

@@ -8,7 +8,7 @@ namespace Tests.BasicTests
     public class TwoClientTests : DistTest
     {
         [Test]
-        public void TwoClientTest()
+        public void TwoClientsOnePodTest()
         {
             var group = SetupCodexNodes(2);
 
@@ -18,6 +18,15 @@ namespace Tests.BasicTests
             PerformTwoClientTest(primary, secondary);
         }
 
+        [Test]
+        public void TwoClientsTwoPodsTest()
+        {
+            var primary = SetupCodexNode();
+            var secondary = SetupCodexNode();
+
+            PerformTwoClientTest(primary, secondary);
+        }
+
         [Test]
         public void TwoClientsTwoLocationsTest()
         {

Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -6,36 +6,15 @@ namespace Tests.DownloadConnectivityTests
     [TestFixture]
     public class FullyConnectedDownloadTests : AutoBootstrapDistTest
     {
-        [Test]
-        public void MetricsDoesNotInterfereWithPeerDownload()
-        {
-            SetupCodexNodes(2, s => s.EnableMetrics());
-
-            AssertAllNodesConnected();
-        }
-
-        [Test]
-        public void MarketplaceDoesNotInterfereWithPeerDownload()
-        {
-            SetupCodexNodes(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens()));
-
-            AssertAllNodesConnected();
-        }
-
         [Test]
         [Combinatorial]
         public void FullyConnectedDownloadTest(
             [Values(1, 3, 5)] int numberOfNodes,
             [Values(1, 10)] int sizeMBs)
         {
-            SetupCodexNodes(numberOfNodes);
+            for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
 
-            AssertAllNodesConnected(sizeMBs);
-        }
-
-        private void AssertAllNodesConnected(int sizeMBs = 10)
-        {
-            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
+            PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
         }
     }
 }

Tests/MembershipChangeTest/BasicTests.cs

@@ -0,0 +1,130 @@
+using DistTestCore;
+using KubernetesWorkflow;
+using NUnit.Framework;
+namespace Tests.MembershipChangeTests
+{
+    [TestFixture]
+    public class MembershipChangeTests : DistTest
+    {
+
+        [Test]
+        public void SingleDownloadWhileAdding()
+        {
+            var filesize = 100.MB();
+
+            var group = SetupCodexNodes(1);
+            var toAdd = SetupCodexNodes(5);
+            var host = SetupCodexNodes(1)[0];
+
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+
+            var testFile = GenerateTestFile(filesize);
+            var contentId = host.UploadFile(testFile);
+            var list = new List<Task>();
+            foreach (var node in toAdd)
+            {
+                list.Add(Task.Run(() => { host.ConnectToPeer(node); }));
+            }
+
+            var resFile = group[0].DownloadContent(contentId);
+            Task.WaitAll(list.ToArray());
+            testFile.AssertIsEqual(resFile);
+        }
+        [Test]
+        public void SingleUploadWhileAdding()
+        {
+            var filesize = 100.MB();
+
+            var group = SetupCodexNodes(1);
+            var toAdd = SetupCodexNodes(5);
+            var host = SetupCodexNodes(1)[0];
+
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+
+            var testFile = GenerateTestFile(filesize);
+            var list = new List<Task>();
+            foreach (var node in toAdd)
+            {
+                list.Add(Task.Run(() => { host.ConnectToPeer(node); }));
+            }
+            var contentId = host.UploadFile(testFile);
+
+
+            Task.WaitAll(list.ToArray());
+            var resFile = group[0].DownloadContent(contentId);
+            testFile.AssertIsEqual(resFile);
+        }
+        [Test]
+        public void SingleDownloadMixedMembership()
+        {
+            var filesize = 100.MB();
+
+            var group = SetupCodexNodes(1);
+            var toAdd = SetupCodexNodes(5);
+            var toRemove = SetupCodexNodes(5);
+            var host = SetupCodexNodes(1)[0];
+
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+            foreach (var node in toRemove)
+            {
+                host.ConnectToPeer(node);
+            }
+
+
+            var testFile = GenerateTestFile(filesize);
+            var contentId = host.UploadFile(testFile);
+
+            for (var i = 0; i < toAdd.Count(); i++)
+            {
+                Task.Run(() => { host.ConnectToPeer(toAdd[i]); });
+                Task.Run(() => { toRemove[i].BringOffline(); });
+            }
+
+            var resFile = group[0].DownloadContent(contentId);
+            testFile.AssertIsEqual(resFile);
+        }
+        [Test]
+        public void SingleUploadMixedMembership()
+        {
+            var filesize = 100.MB();
+
+            var group = SetupCodexNodes(1);
+            var toAdd = SetupCodexNodes(5);
+            var toRemove = SetupCodexNodes(5);
+            var host = SetupCodexNodes(1)[0];
+
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+            foreach (var node in toRemove)
+            {
+                host.ConnectToPeer(node);
+            }
+
+            var testFile = GenerateTestFile(filesize);
+
+            var list = new List<Task>();
+
+            for (var i = 0; i < toAdd.Count(); i++)
+            {
+                Task.Run(() => { host.ConnectToPeer(toAdd[i]); });
+                Task.Run(() => { toRemove[i].BringOffline(); });
+            }
+            var contentId = host.UploadFile(testFile);
+
+            var resFile = group[0].DownloadContent(contentId);
+            testFile.AssertIsEqual(resFile);
+        }
+    }
+
+}

Tests/MembershipChangeTest/DownloadTests.cs

@@ -0,0 +1,67 @@
+using DistTestCore;
+using KubernetesWorkflow;
+using NUnit.Framework;
+namespace Tests.MembershipChangeTests
+{
+    [TestFixture]
+    public class DownloadMembershipChangeTests : DistTest
+    {
+        [TestCase(1, 100, 5, 0)]
+        [TestCase(1, 100, 0, 5)]
+        [TestCase(1, 100, 5, 5)]
+        [UseLongTimeouts]
+        public void DownloadMembershipChange(int numberOfNodes, int filesize, int numberOfNodesToAdd = 0, int numberOfNodesToRemove = 0)
+        {
+            // Setup 3 node groups, one which will be added during the procedure, one which will be dropped during the procedure, and the one being tested. 
+            ICodexNodeGroup? toAdd = null;
+            ICodexNodeGroup? toRemove = null;
+            var group = SetupCodexNodes(numberOfNodes);
+            if (numberOfNodesToAdd != 0)
+                toAdd = SetupCodexNodes(numberOfNodesToAdd);
+            if (numberOfNodesToRemove != 0)
+                toRemove = SetupCodexNodes(numberOfNodesToRemove);
+            var host = SetupCodexNodes(1)[0];
+
+            // Connect the main and dropping nodes to the host
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+            if (toRemove != null)
+                foreach (var node in toRemove)
+                    host.ConnectToPeer(node);
+
+            // Upload a file to the host
+            var testFile = GenerateTestFile(filesize.MB());
+            var contentId = host.UploadFile(testFile);
+
+            var list = new List<Task<TestFile?>>();
+                        
+            // Start the download for each node
+            foreach (var node in group)
+            {
+                list.Add(Task.Run(() => { return node.DownloadContent(contentId); }));
+            }
+
+            // Start adding and dropping nodes during the download
+            // TODO: The log is put here for debug, but without it the members do not run async
+            for (var i = 0; (toAdd != null && i < toAdd.Count()) || (toRemove != null && i < toRemove.Count()); i++)
+            {
+                Log($"Iteration {i}");
+                if (toAdd != null && i < toAdd.Count())
+                    Task.Run(() => { host.ConnectToPeer(toAdd[i]); });
+                if (toRemove != null && i < toRemove.Count())
+                    Task.Run(() => { toRemove[i].BringOffline(); });
+            }
+
+            // Wait for the download to finish
+            Task.WaitAll(list.ToArray());
+
+            // Assert that the download was successful
+            foreach (var task in list)
+            {
+                testFile.AssertIsEqual(task.Result);
+            }
+        }
+    }
+}

Tests/MembershipChangeTest/MixedTests.cs

@@ -0,0 +1,83 @@
+using DistTestCore;
+using KubernetesWorkflow;
+using NUnit.Framework;
+namespace Tests.MembershipChangeTests
+{
+    [TestFixture]
+    public class MixedMembershipChangeTests : DistTest
+    {
+        [TestCase(1, 5, 0, 0)]
+
+        [UseLongTimeouts]
+        public void MixedMembershipChange(int numberOfNodes, int filesize, int numberOfNodesToAdd, int numberOfNodesToRemove)
+        {
+            // Creating the node groups
+            ICodexNodeGroup? toAdd = null;
+            ICodexNodeGroup? toAddSecondary = null;
+            ICodexNodeGroup? toRemove = null;
+            var group = SetupCodexNodes(numberOfNodes);
+            
+            if (numberOfNodesToAdd != 0) {
+                toAdd = SetupCodexNodes(numberOfNodesToAdd);
+                toAddSecondary = SetupCodexNodes(numberOfNodesToAdd);
+            }
+            if (numberOfNodesToRemove != 0)
+                toRemove = SetupCodexNodes(numberOfNodesToRemove);
+            var host = SetupCodexNodes(1)[0];
+
+            // Connect the main and dropping nodes to the host
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+            if (toRemove != null)
+                foreach (var node in toRemove)
+                    host.ConnectToPeer(node);
+
+            var testFile = GenerateTestFile(filesize.MB());
+            var contentId = host.UploadFile(testFile);
+
+            var testfiles = new List<TestFile>();
+            var contentIds = new List<Task<ContentId>>();
+
+            var downloads = new List<Task<TestFile?>>();
+
+            // Start adding and dropping nodes
+            for (var i = 0; (toAdd != null && i < toAdd.Count()) || (toRemove != null && i < toRemove.Count()); i++)
+            {
+                Log($"Iteration {i}");
+                if (toAdd != null && i < toAdd.Count())
+                    Task.Run(() => { host.ConnectToPeer(toAdd[i]); });
+                if (toRemove != null && i < toRemove.Count())
+                    Task.Run(() => { toRemove[i].BringOffline(); });
+            }
+
+            // Start the upload for each node in the main group
+            for (int i = 0; i < group.Count(); i++)
+            {
+                testfiles.Add(GenerateTestFile(filesize.MB()));
+                var n = i;
+                contentIds.Add(Task.Run(() => { return host.UploadFile(testfiles[n]); }));
+            }
+
+            // Start the download for each node in the main group
+            for (int i = 0; i < group.Count(); i++)
+            {
+                var n = i;
+                downloads.Add(Task.Run(() => { return group[n].DownloadContent(contentId); }));
+            }
+
+            // Wait for the download to finish
+            Task.WaitAll(downloads.ToArray());
+
+            // Wait for the upload to finish
+            Task.WaitAll(contentIds.ToArray());
+
+            // Assert that the files are intact
+            for (int i = 0; i < group.Count(); i++)
+            {
+                testfiles[i].AssertIsEqual(downloads[i].Result);
+            }
+        }
+    }
+}

Tests/MembershipChangeTest/UploadTests.cs

@@ -0,0 +1,78 @@
+using DistTestCore;
+using KubernetesWorkflow;
+using NUnit.Framework;
+namespace Tests.MembershipChangeTests
+{
+     [TestFixture]
+    public class UploadMembershipChangeTests : DistTest
+    {
+        [TestCase(1, 100, 5, 0)]
+        [TestCase(1, 100, 0, 5)]
+        [TestCase(1, 100, 5, 5)]
+
+        [UseLongTimeouts]
+        public void UploadMembershipChange(int numberOfNodes, int filesize, int numberOfNodesToAdd = 0, int numberOfNodesToRemove = 0)
+        {
+            // Creating the node groups
+            ICodexNodeGroup? toAdd = null;
+            ICodexNodeGroup? toRemove = null;
+            var group = SetupCodexNodes(numberOfNodes);
+            if (numberOfNodesToAdd != 0)
+                toAdd = SetupCodexNodes(numberOfNodesToAdd);
+            if (numberOfNodesToRemove != 0)
+                toRemove = SetupCodexNodes(numberOfNodesToRemove);
+            var host = SetupCodexNodes(1)[0];
+
+            // Connect the main and dropping nodes to the host
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+            if (toRemove != null)
+                foreach (var node in toRemove)
+                    host.ConnectToPeer(node);
+
+
+            var testfiles = new List<TestFile>();
+            var contentIds = new List<Task<ContentId>>();
+
+            // Start adding and dropping nodes
+
+            // Start the upload for each node in the main group
+            for (int i = 0; i < group.Count(); i++)
+            {
+                testfiles.Add(GenerateTestFile(filesize.MB()));
+                var n = i;
+                contentIds.Add(Task.Run(() => { return host.UploadFile(testfiles[n]); }));
+            }
+            for (var i = 0; (toAdd != null && i < toAdd.Count()) || (toRemove != null && i < toRemove.Count()); i++)
+            {
+                Log($"Iteration {i}");
+                if (toAdd != null && i < toAdd.Count())
+                    Task.Run(() => { host.ConnectToPeer(toAdd[i]); });
+                if (toRemove != null && i < toRemove.Count())
+                    Task.Run(() => { toRemove[i].BringOffline(); });
+            }
+
+            // Wait for the upload to finish
+            Task.WaitAll(contentIds.ToArray());
+
+            // Download the files
+            var downloads = new List<Task<TestFile?>>();
+            for (int i = 0; i < group.Count(); i++)
+            {
+                var n = i;
+                downloads.Add(Task.Run(() => { return group[n].DownloadContent(contentIds[n].Result); }));
+            }
+
+            // Wait for the download to finish
+            Task.WaitAll(downloads.ToArray());
+
+            // Assert that the files are intact
+            for (int i = 0; i < group.Count(); i++)
+            {
+                testfiles[i].AssertIsEqual(downloads[i].Result);
+            }
+        }
+    }
+}

Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs

@@ -1,4 +1,5 @@
 using DistTestCore;
+using DistTestCore.Helpers;
 using NUnit.Framework;
 
 namespace Tests.PeerDiscoveryTests
@@ -46,7 +47,7 @@ namespace Tests.PeerDiscoveryTests
 
         private void AssertAllNodesConnected()
         {
-            CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
+            PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
         }
     }
 }

Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -1,4 +1,5 @@
 using DistTestCore;
+using DistTestCore.Helpers;
 using NUnit.Framework;
 
 namespace Tests.PeerDiscoveryTests
@@ -16,36 +17,23 @@ namespace Tests.PeerDiscoveryTests
             Assert.That(result.IsPeerFound, Is.False);
         }
 
-        [Test]
-        public void MetricsDoesNotInterfereWithPeerDiscovery()
-        {
-            SetupCodexNodes(2, s => s.EnableMetrics());
-
-            AssertAllNodesConnected();
-        }
-
-        [Test]
-        public void MarketplaceDoesNotInterfereWithPeerDiscovery()
-        {
-            SetupCodexNodes(2, s => s.EnableMarketplace(1000.TestTokens()));
-
-            AssertAllNodesConnected();
-        }
-
         [TestCase(2)]
         [TestCase(3)]
         [TestCase(10)]
         [TestCase(20)]
-        public void VariableNodes(int number)
+        public void VariableNodesInPods(int number)
         {
-            SetupCodexNodes(number);
+            for (var i = 0; i < number; i++)
+            {
+                SetupCodexNode();
+            }
 
             AssertAllNodesConnected();
         }
 
         private void AssertAllNodesConnected()
         {
-            CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
+            PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
         }
     }
 }

Tests/Tests.csproj

@@ -13,7 +13,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\ContinuousTests\ContinuousTests.csproj" />
     <ProjectReference Include="..\DistTestCore\DistTestCore.csproj" />
   </ItemGroup>
 

docker/README.md

@@ -0,0 +1,138 @@
+# Run tests with Docker in Kubernetes
+
+ We may [run tests on local](../LOCALSETUP.MD) or remote Kubernetes cluster. Local cluster flow uses direct access to nodes ports and this is why we introduced a different way to check services ports, for more information please see [Tests run modes](../../../issues/20). Configuration option `RUNNERLOCATION` is responsible for that.
+
+
+ For local run it is easier to install .Net and run tests on Docker Desktop Kubernetes cluster. In case of remote run we do not expose services via Ingress Controller and we can't access cluster nodes, this is why we should run tests only inside the Kubernetes.
+
+ We can run tests on remote cluster in the following ways
+
+#### Run pod inside the cluster using generic .Net image
+
+<details>
+<summary>steps</summary>
+
+1. Create dist-tests-runner.yaml
+   ```yaml
+      --
+      apiVersion: v1
+      kind: Pod
+      metadata:
+      name: dist-tests-runner
+      namespace: default
+      spec:
+      containers:
+      - name: dotnet
+         image: mcr.microsoft.com/dotnet/sdk:7.0
+         command: ["sleep", "infinity"]
+   ```
+2. Deploy pod in the cluster
+   ```shell
+   kubectl apply -f dist-tests-runner.yaml
+   ```
+
+3. Copy kubeconfig to the pod
+   ```shell
+   kubectl cp kubeconfig.yaml dist-tests-runner:/opt
+   ```
+
+4. Exec into the pod via kubectl or [OpenLens](https://github.com/MuhammedKalkan/OpenLens)
+   ```shell
+   kubectl exec -it dist-tests-runner -- bash
+   ```
+
+5. Clone repository inside the pod
+   ```shell
+   git clone https://github.com/codex-storage/cs-codex-dist-tests.git
+   ```
+
+6. Update kubeconfig option in config file
+   ```shell
+   cd cs-codex-dist-tests
+   vi DistTestCore/Configuration.cs
+   ```
+   ```dotnet
+   GetNullableEnvVarOrDefault("KUBECONFIG", "/opt/kubeconfig.yaml")
+   ```
+
+7. Run tests
+   ```shell
+   dotnet test Tests
+   ```
+
+8. Check the results and analyze the logs
+</details>
+
+#### Run pod inside the cluster using [prepared Docker images](https://hub.docker.com/r/codexstorage/cs-codex-dist-tests/tags)
+
+ Before the run we should create some objects inside the cluster
+ 1. Namespace where we will run the image
+ 2. [Service Account to run tests inside the cluster](https://github.com/codex-storage/cs-codex-dist-tests/issues/21)
+ 3. Secret with kubeconfig for created SA
+ 4. Configmap with custom app config if required
+
+ For more information please see [Manual run inside Kubernetes via Job](../../../issues/7)
+
+ Then we need to create a manifest to run the pod
+ <details>
+ <summary>runner.yaml</summary>
+
+ ```yaml
+ ---
+ apiVersion: v1
+ kind: Pod
+ metadata:
+   name: dist-tests-runner
+   namespace: cs-codex-dist-tests
+   labels:
+     name: cs-codex-dist-tests
+ spec:
+   containers:
+   - name: cs-codex-dist-tests
+     image: codexstorage/cs-codex-dist-tests:sha-671ee4e
+     env:
+     - name: RUNNERLOCATION
+       value: InternalToCluster
+     - name: KUBECONFIG
+       value: /opt/kubeconfig.yaml
+     - name: CONFIG
+       value: "/opt/Configuration.cs"
+     - name: CONFIG_SHOW
+       value: "true"
+     volumeMounts:
+     - name: kubeconfig
+       mountPath: /opt/kubeconfig.yaml
+       subPath: kubeconfig.yaml
+     - name: config
+       mountPath: /opt/Configuration.cs
+       subPath: Configuration.cs
+     - name: logs
+       mountPath: /var/log/cs-codex-dist-tests
+     # command:
+     # - "dotnet"
+     # - "test"
+     # - "Tests"
+   restartPolicy: Never
+   volumes:
+     - name: kubeconfig
+       secret:
+         secretName: cs-codex-dist-tests-app-kubeconfig
+     - name: config
+       configMap:
+         name: cs-codex-dist-tests
+     - name: logs
+       hostPath:
+         path: /var/log/cs-codex-dist-tests
+ ```
+ For more information about pod variables please see [job.yaml](job.yaml).
+ </details>
+
+ And then apply it
+ ```shell
+ kubectl apply -f runner.yaml
+ ```
+
+ After the pod run, custom [entrypoint](docker-entrypoint.sh) will do the following
+ 1. Clone repository
+ 2. Switch to the specific branch - `master` by default
+ 3. Run all tests - `dotnet test`

docker/docker-compose.yaml

@@ -1,11 +0,0 @@
-services:
-  dist-test-run:
-    build:
-      context: ..
-      dockerfile: docker/Dockerfile
-    environment:
-      - CODEXDOCKERIMAGE=codexstorage/nim-codex:sha-14c5270
-      - BRANCH="feature/docker-image-testruns"
-      - KUBECONFIG=/opt/kubeconfig
-      - LOGPATH=/opt/logs
-      - RUNNERLOCATION=ExternalToCluster

docker/job.yaml

@@ -14,8 +14,7 @@ spec:
     spec:
       containers:
       - name: ${NAMEPREFIX}-runner
-        image: codexstorage/cs-codex-dist-tests:latest
-        imagePullPolicy: Always
+        image: codexstorage/cs-codex-dist-tests:sha-300b91e
         env:
         - name: RUNNERLOCATION
           value: InternalToCluster
@@ -29,10 +28,6 @@ spec:
           value: ${BRANCH}
         - name: SOURCE
           value: ${SOURCE}
-        - name: RUNID
-          value: ${RUNID}
-        - name: TESTID
-          value: ${TESTID}
         volumeMounts:
         - name: kubeconfig
           mountPath: /opt/kubeconfig.yaml

docs/AUTOMATION.md

@@ -1,253 +0,0 @@
-# Distributed Tests automation
-
- 1. [Description](#description)
- 2. [Architecture](#architecture)
- 3. [Run on remote cluster](#run-on-remote-cluster)
-    - [Kubernetes cluster pre-configuration](#kubernetes-cluster-pre-configuration)
-    - [Run tests manually](#run-tests-manually)
-    - [Run tests automatically](#run-tests-automatically)
-    - [Tests logs](#tests-logs)
-
-
-## Description
-
- The goal of [Distributed System Tests for Nim-Codex](../../) is to test how [Codex](https://codex.storage) works in different topologies in the distributed network and to be able to detect regressions during development.
-
- We can [run Tests locally](LOCALSETUP.md) and it works well, but in order to scale that we may need to run Tests in an automatic way using remote Kubernetes cluster.
-
- Initially, we are considering to run dist-tests on [nim-codex](https://github.com/codex-storage/nim-codex) master branch merge, to be able to determine regressions. And we also working on [Continuous Tests](/ContinuousTests) which are called to detect issues on continuous Codex runs.
-
-
-## Architecture
-
-<img src="Architecture.png" alt="Architecture" width="800"/>
-
-```
-                                        Logs     --> Kibana
-                                      /                |
-GitHub --> CI --> Kubernetes -->  Job    Prometheus  Elaticsearch
-      \                          /    \  /         \   |
-        ------------------------        Metrics  --> Grafana
-```
-
-
-### Components
-
-| Component                               | Description                             |
-| --------------------------------------- | --------------------------------------- |
-| [cs-codex-dist-tests](/)                | Distributed System Tests                |
-| [Kubernetes](https://kubernetes.io)     | Environment where we run Tests          |
-| [Vector](https://vector.dev)            | Ship logs to the Elasticsearch          |
-| [Elasticsearch](https://www.elastic.co) | Store and index logs                    |
-| [Kibana](https://www.elastic.co)        | Discover the logs in Elasticsearch      |
-| [Grafana](https://grafana.com)          | Visualize tests run results and metrics |
-| [Prometheus](https://prometheus.io)     | Collect and store metrics               |
-
-> Note: These components are not mentioned on the diagram and provided to understand what do we have under the hood
-
-
-## Run on remote cluster
-
- In case of local run we use [Docker Desktop Kubernetes cluster](https://docs.docker.com/desktop/kubernetes/) and during the services checks, app connect directly to the cluster worker nodes and perform ports check. And in case of remote cluster, it would be required to configure services ports exposing using Ingress Controller or run tests directly inside the cluster.
-
- Now, we have a configuration key `RUNNERLOCATION` which change the logic of the services ports check and when we run tests inside remote cluster we should set it to the `InternalToCluster`.
-
- As for now, it was decided to run tests inside the remote Kubernetes cluster using CI because
- - Stable connection from app to the Kubernetes API
- - Independence from the CI limitations for long runs
- - Easy, fast, configurable and detachable run
-
- Because tests are run on remote cluster we need a way to see their execution status and to analyze the logs as well. For that we use Elasticsearch, Kibana and Grafana with logs shipped by Vector. Please see [Tests logs](#tests-logs) for more information.
-
- Now we can [Run tests manually](#run-tests-manually) and [Run tests automatically](#run-tests-automatically) on [remote Kubernetes cluster which requires to be pre-configured](#kubernetes-cluster-pre-configuration).
-
-
-### Kubernetes cluster pre-configuration
-
- Before running the tests on remote Kubernetes cluster we performed [manual pre-configuration](../../../issues/7) and it was require to
- 1. [Create a namespace](../../../issues/7)
- 2. [Create kubeconfig for App](../../../issues/21)
- 3. [Create a secret with created kubeconfig](../../../issues/21)
- 4. [Create kubeconfig for GitHub Actions](../../../issues/19)
-
-
-### Run tests manually
-
- **To run tests manually we have the following requirements**
- 1. Get kubeconfig - to access the cluster
- 2. Install [kubectl](https://kubernetes.io/docs/tasks/tools/) - to create resources in the cluster
- 3. Install [OpenLens](https://github.com/MuhammedKalkan/OpenLens) - to browse cluster resources
-
- **And to run the tests we should perform the following steps**
- 1. Create a Pod in the cluster, in the `default` namespace and consider to use your own value for `metadata.name`
-    <details>
-    <summary>pod.yaml</summary>
-
-    ```yaml
-    ---
-    apiVersion: v1
-    kind: Pod
-    metadata:
-      name: dist-tests-runner
-      namespace: default
-      labels:
-        app: dist-tests-runner
-        launched: manually
-    spec:
-      containers:
-      - name: dotnet
-        image: mcr.microsoft.com/dotnet/sdk:7.0
-        env:
-        - name: RUNNERLOCATION
-          value: InternalToCluster
-        - name: KUBECONFIG
-          value: /opt/kubeconfig.yaml
-        command: ["sleep", "infinity"]
-        volumeMounts:
-        - name: kubeconfig
-          mountPath: /opt/kubeconfig.yaml
-          subPath: kubeconfig.yaml
-        - name: logs
-          mountPath: /var/log/cs-codex-dist-tests
-      restartPolicy: Never
-      volumes:
-        - name: kubeconfig
-          secret:
-            secretName: cs-codex-dist-tests-app-kubeconfig
-        - name: logs
-          hostPath:
-            path: /var/log/cs-codex-dist-tests
-    ```
-    </details>
-
-    ```shell
-    kubectl apply -f pod.yaml
-    ```
-
- 2. Exec into the Pod using the name you set in the previous step
-    ```shell
-    # kubectl
-    kubectl exec -it dist-tests-runner -- bash
-
-    # OpenLens
-    OpenLens --> Pods --> dist-tests-runner --> "Press on it" --> Pod Shell
-    ```
-
- 3. Clone dist-tests repository
-    ```shell
-    folder="/opt/dist-tests"
-
-    git clone https://github.com/codex-storage/cs-codex-dist-tests.git $folder
-    cd $folder
-    ```
-
- 4. Define variables - optional
-    ```shell
-    # RUNNERLOCATION                              # defined at Pod creation
-    # KUBECONFIG                                  # defined at Pod creation
-    export LOGPATH="/var/log/cs-codex-dist-tests" # Logs from that location will be send in Elasticsearch
-    export RUNID=$(date +%Y%m%d-%H%M%S)           # Run ID to show in Kibana/Grafana
-    export TESTID=$(git rev-parse --short HEAD)   # Test ID to show in Kibana/Grafana
-    ```
-
- 5. Run tests
-    ```shell
-    # All tests
-    dotnet test
-
-    # Short tests
-    dotnet test Tests
-
-    # Long tests
-    dotnet test LongTests
-
-    # Specific test
-    dotnet test --filter=CodexLogExample
-    ```
-
- 6. We can see in OpenLens Pods started by dist-tests app
-
- 7. If we set `LOGPATH` to a location specified above, we should be able to see tests execution status in Kibana/Grafana. For more information, please see [Tests logs](#tests-logs).
-
-
-### Run tests automatically
-
- Now we use GitHub Actions to trigger dist-tests run manually and considering to run them on [nim-codex](https://github.com/codex-storage/nim-codex) master branch merge.
-
- **It works in the following way**
- 1. Github Actions secrets contains [kubeconfig to interact with the Kubernetes cluster](../../../issues/19).
- 2. GitHub Actions [workflow](../../../actions/workflows/dist-tests.yaml) uses kubectl to create Kubernetes Job based on the [job.yaml](/docker/job.yaml) manifest file. It also accept optional inputs at run
-    - `source`     - Dist-tests source repository
-    - `branch`     - Repository branch
-    - `namespace`  - Namespace where Dist-test runner will de created
-    - `nameprefix` - Dist-test runner name prefix
-
- 3. Kubernetes Job will run the Pod with a custom [Docker image](/docker/Dockerfile) - [codexstorage/cs-codex-dist-tests](https://hub.docker.com/r/codexstorage/cs-codex-dist-tests/tags).  
-  Image [entrypoint](/docker/docker-entrypoint.sh) is customizable and we can pass the following variables
-    - `SOURCE` - Dist-tests source repository, useful when we work with forks - `default="current repository"`
-    - `BRANCH` - Repository branch, useful when we would like to run tests from the custom branch - `default="master"`
-    - `FOLDER` - Where to clone repository and it is done just to organize the things - `default="/opt/dist-tests"`
-
-    > **Note:** Variables `SOURCE` and `BRANCH` passed by GitHub Actions to the Kubernetes Job and then to the Pod.
-
- 4. Job manifest is setting all required variables which are part of the Pod run
-    ```shell
-    RUNNERLOCATION=InternalToCluster
-    KUBECONFIG=/opt/kubeconfig.yaml
-    LOGPATH=/var/log/cs-codex-dist-tests
-    NAMESPACE=default
-    SOURCE=current repository
-    BRANCH=master
-    RUNID=datetime
-    TESTID=short sha
-    ```
-
- 5. Dist-tests runner will use [kubeconfig to interact with the Kubernetes and run the tests](../../../issues/21), which is set by `KUBECONFIG` variable.
-
- 6. Runner will execute all tests and will write the logs to the `/var/log/cs-codex-dist-tests` folder.
-
- 7. Kubernetes Job status will be changed from `Running` to the `Completed` or `Failed`.
-
- 8. Vector will ship the logs to the Elasticsearch.
-
- 9. We can check execution status in Kibana and Grafana. For more information, please see [Tests logs](#tests-logs).
-
-
-### Tests logs
-
- > **Note:** This part is not finished yet and it is under development
-
- We use Elasticsearch to store and discover the logs of the tests execution and Codex Pods which are run during the tests.
-
- We can use [Kibana](#kibana) to discover all the logs and [Grafana](#grafana) to see tests execution status.
-
-
-#### Endpoints
-
- | App     | URL                                                                          | Authentication |
- | ------- | ---------------------------------------------------------------------------- | -------------- |
- | Kibana  | [kibana.dist-tests.codex.storage](https://kibana.dist-tests.codex.storage)   | GitHub account |
- | Grafana | [grafana.dist-tests.codex.storage](https://grafana.dist-tests.codex.storage) | GitHub account |
-
-
-#### Kibana
-
- As for now, we have the following indices in Kibana
- | # | Index Pattern              | Description                         |
- | - | -------------------------- | ----------------------------------- |
- | 1 | `dist-tests-status*`       | Dist-tests execution status logs    |
- | 2 | `dist-tests-logs*`         | Dist-tests logs collected by runner |
- | 3 | `dist-tests-pods*`         | Dist-tests Pods logs                |
- | 4 | `continuous-tests-status*` | Dist-tests execution status logs    |
- | 5 | `continuous-tests-logs*`   | Dist-tests logs collected by runner |
- | 6 | `continuous-tests-pods*`   | Dist-tests Pods logs                |
- | 7 | `kubernetes*`              | All Kubernetes Pods logs            |
-
-
-#### Grafana
-
- As for now, we have the following Dashboards in Grafana
- | # | Dashboard                  |
- | - | -------------------------- |
- | 1 | Distributed Tests - Status |
- | 2 | Continuous Tests - Status  |