Uncommented tests

.github/workflows/dist-tests.yaml

@@ -15,19 +15,19 @@ on:
   workflow_dispatch:
     inputs:
       branch:
-        description: Branch (master)
+        description: Branch
         required: false
         type: string
       source:
-        description: Repository with tests (current)
+        description: Repository with tests
         required: false
         type: string
       nameprefix:
-        description: Runner prefix (cs-codex-dist-tests)
+        description: Runner job/pod name prefix
         required: false
         type: string
       namespace:
-        description: Runner namespace (cs-codex-dist-tests)
+        description: Kubernetes namespace for runner
         required: false
         type: string
 
@@ -56,8 +56,6 @@ jobs:
           [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
           [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
           [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
-          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
-          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
 
       - name: Kubectl - Install ${{ env.KUBE_VERSION }}
         uses: azure/setup-kubectl@v3
@@ -71,4 +69,5 @@ jobs:
 
       - name: Kubectl - Create Job
         run: |
+          export RUNID=$(date +%Y%m%d-%H%M%S)
           envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -

.github/workflows/docker-reusable.yml

@@ -1,177 +0,0 @@
-name: Docker - Reusable
-
-
-on:
-  workflow_call:
-    inputs:
-      docker_file:
-        default: docker/Dockerfile
-        description: Dockerfile
-        required: false
-        type: string
-      docker_repo:
-        default: codexstorage/cs-codex-dist-tests
-        description: DockerHub repository
-        required: false
-        type: string
-      tag_latest:
-        default: true
-        description: Set latest tag for Docker images
-        required: false
-        type: boolean
-      tag_sha:
-        default: true
-        description: Set Git short commit as Docker tag
-        required: false
-        type: boolean
-      tag_suffix:
-        default: ''
-        description: Suffix for Docker images tag
-        required: false
-        type: string
-
-
-env:
-  DOCKER_FILE: ${{ inputs.docker_file }}
-  DOCKER_REPO: ${{ inputs.docker_repo }}
-  TAG_LATEST: ${{ inputs.tag_latest }}
-  TAG_SHA: ${{ inputs.tag_sha }}
-  TAG_SUFFIX: ${{ inputs.tag_suffix }}
-
-
-jobs:
-  # Build platform specific image
-  build:
-    strategy:
-      fail-fast: true
-      matrix:
-        target:
-          - os: linux
-            arch: amd64
-          - os: linux
-            arch: arm64
-        include:
-          - target:
-              os: linux
-              arch: amd64
-            builder: ubuntu-22.04
-          - target:
-              os: linux
-              arch: arm64
-            builder: buildjet-4vcpu-ubuntu-2204-arm
-
-    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
-    runs-on: ${{ matrix.builder }}
-    env:
-      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.DOCKER_REPO }}
-
-      - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Build and Push by digest
-        id: build
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          file: ${{ env.DOCKER_FILE }}
-          platforms: ${{ env.PLATFORM }}
-          push: true
-          labels: ${{ steps.meta.outputs.labels }}
-          outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
-
-      - name: Docker - Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-
-      - name: Docker - Upload digest
-        uses: actions/upload-artifact@v3
-        with:
-          name: digests
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
-
-
-  # Publish multi-platform image
-  publish:
-    name: Publish multi-platform image
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Docker - Variables
-        run: |
-          # Adjust custom suffix when set and
-          if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
-            echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
-          fi
-          # Disable SHA tags on tagged release
-          if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
-            echo "TAG_SHA=false" >>$GITHUB_ENV
-          fi
-          # Handle latest and latest-custom using raw
-          if [[ ${{ env.TAG_SHA }} == "false" ]]; then
-            echo "TAG_LATEST=false" >>$GITHUB_ENV
-            echo "TAG_RAW=true" >>$GITHUB_ENV
-            if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
-              echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
-            else
-              echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
-            fi
-          else
-            echo "TAG_RAW=false" >>$GITHUB_ENV
-          fi
-
-      - name: Docker - Download digests
-        uses: actions/download-artifact@v3
-        with:
-          name: digests
-          path: /tmp/digests
-
-      - name: Docker - Set up Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Docker - Meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ${{ env.DOCKER_REPO }}
-          flavor: |
-            latest=${{ env.TAG_LATEST }}
-            suffix=${{ env.TAG_SUFFIX }},onlatest=true
-          tags: |
-            type=semver,pattern={{version}}
-            type=raw,enable=${{ env.TAG_RAW }},value=latest
-            type=sha,enable=${{ env.TAG_SHA }}
-
-      - name: Docker - Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker - Create manifest list and push
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
-
-      - name: Docker - Inspect image
-        run: |
-          docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}

.github/workflows/docker.yml

@@ -11,12 +11,113 @@ on:
       - docker/Dockerfile
       - docker/docker-entrypoint.sh
       - .github/workflows/docker.yml
-      - .github/workflows/docker-reusable.yml
   workflow_dispatch:
 
 
+env:
+  DOCKER_FILE: docker/Dockerfile
+  DOCKER_REPO: codexstorage/cs-codex-dist-tests
+
+
 jobs:
-  build-and-push:
-    name: Build and Push
-    uses: ./.github/workflows/docker-reusable.yml
-    secrets: inherit
+  # Build platform specific image
+  build:
+    strategy:
+      fail-fast: true
+      matrix:
+        target:
+          - os: linux
+            arch: amd64
+          - os: linux
+            arch: arm64
+        include:
+          - target:
+              os: linux
+              arch: amd64
+            builder: ubuntu-22.04
+          - target:
+              os: linux
+              arch: arm64
+            builder: buildjet-4vcpu-ubuntu-2204-arm
+
+    name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
+    runs-on: ${{ matrix.builder }}
+    outputs:
+      tags-linux-amd64: ${{ steps.tags.outputs.tags-linux-amd64 }}
+      tags-linux-arm64: ${{ steps.tags.outputs.tags-linux-arm64 }}
+    env:
+      PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
+      SUFFIX: ${{ format('{0}-{1}', 'linux', matrix.target.arch) }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_REPO }}
+          flavor: |
+            latest=false
+          tags: |
+            type=semver,pattern={{version}},suffix=-${{ env.SUFFIX }}
+            type=sha,suffix=-${{ env.SUFFIX }},enable=${{ !startsWith(github.ref, 'refs/tags/') }}
+
+      - name: Docker - Set tags output
+        id: tags
+        run: |
+          if [[ '${{ matrix.target.os }}' == 'linux' && '${{ matrix.target.arch }}' == 'amd64' ]]; then
+            echo "tags-linux-amd64=${{ steps.meta.outputs.tags }}" >> "$GITHUB_OUTPUT"
+          elif [[ '${{ matrix.target.os }}' == 'linux' && '${{ matrix.target.arch }}' == 'arm64' ]]; then
+            echo "tags-linux-arm64=${{ steps.meta.outputs.tags }}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Build and Push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ${{ env.DOCKER_FILE }}
+          platforms: ${{ env.PLATFORM }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  # Publish single image
+  publish:
+    name: Push single image
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Docker - Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.DOCKER_REPO }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=sha,enable=${{ !startsWith(github.ref, 'refs/tags/') }}
+
+      - name: Docker - Set tags
+        run: |
+          # Transform multi-line tags in to the comma-seperated
+          TAGS=$(echo "${{ steps.meta.outputs.tags }}" | tr '\n' ',' | awk '{gsub(/,$/,"");}1')
+          echo "TAGS=${TAGS}" >>$GITHUB_ENV
+
+      - name: Docker - Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker - Create and push manifest images
+        uses: Noelware/docker-manifest-action@master
+        with:
+          inputs: ${{ env.TAGS }}
+          images: ${{ needs.build.outputs.tags-linux-amd64 }},${{ needs.build.outputs.tags-linux-arm64 }}
+          push: true

CodexNetDeployer/CodexNodeStarter.cs

@@ -8,27 +8,28 @@ namespace CodexNetDeployer
     public class CodexNodeStarter
     {
         private readonly Configuration config;
+        private readonly WorkflowCreator workflowCreator;
         private readonly TestLifecycle lifecycle;
         private readonly GethStartResult gethResult;
         private string bootstrapSpr = "";
         private int validatorsLeft;
 
-        public CodexNodeStarter(Configuration config, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
+        public CodexNodeStarter(Configuration config, WorkflowCreator workflowCreator, TestLifecycle lifecycle, GethStartResult gethResult, int numberOfValidators)
         {
             this.config = config;
+            this.workflowCreator = workflowCreator;
             this.lifecycle = lifecycle;
             this.gethResult = gethResult;
             validatorsLeft = numberOfValidators;
         }
 
-        public CodexNodeStartResult? Start(int i)
+        public RunningContainer? Start(int i)
         {
             Console.Write($" - {i} = ");
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var workflow = workflowCreator.CreateWorkflow();
             var workflowStartup = new StartupConfig();
             workflowStartup.Add(gethResult);
             workflowStartup.Add(CreateCodexStartupConfig(bootstrapSpr, i, validatorsLeft));
-            workflowStartup.NameOverride = GetCodexContainerName(i);
 
             var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
 
@@ -61,7 +62,7 @@ namespace CodexNetDeployer
 
                         if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr;
                         validatorsLeft--;
-                        return new CodexNodeStartResult(workflow, container, codexAccess);
+                        return container;
                     }
                 }
             }
@@ -76,12 +77,6 @@ namespace CodexNetDeployer
             return null;
         }
 
-        private string GetCodexContainerName(int i)
-        {
-            if (i == 0) return "BOOTSTRAP";
-            return "CODEX" + i;
-        }
-
         private CodexStartupConfig CreateCodexStartupConfig(string bootstrapSpr, int i, int validatorsLeft)
         {
             var codexStart = new CodexStartupConfig(config.CodexLogLevel);
@@ -91,36 +86,14 @@ namespace CodexNetDeployer
             var marketplaceConfig = new MarketplaceInitialConfig(100000.Eth(), 0.TestTokens(), validatorsLeft > 0);
             marketplaceConfig.AccountIndexOverride = i;
             codexStart.MarketplaceConfig = marketplaceConfig;
-            codexStart.MetricsMode = config.Metrics;
+            codexStart.MetricsEnabled = config.RecordMetrics;
 
             if (config.BlockTTL != Configuration.SecondsIn1Day)
             {
                 codexStart.BlockTTL = config.BlockTTL;
             }
-            if (config.BlockMI != Configuration.TenMinutes)
-            {
-                codexStart.BlockMaintenanceInterval = TimeSpan.FromSeconds(config.BlockMI);
-            }
-            if (config.BlockMN != 1000)
-            {
-                codexStart.BlockMaintenanceNumber = config.BlockMN;
-            }
 
             return codexStart;
         }
     }
-
-    public class CodexNodeStartResult
-    {
-        public CodexNodeStartResult(StartupWorkflow workflow, RunningContainer container, CodexAccess access)
-        {
-            Workflow = workflow;
-            Container = container;
-            Access = access;
-        }
-
-        public StartupWorkflow Workflow { get; }
-        public RunningContainer Container { get; }
-        public CodexAccess Access { get; }
-    }
 }

CodexNetDeployer/Configuration.cs

@@ -1,13 +1,12 @@
 using ArgsUniform;
+using DistTestCore;
 using DistTestCore.Codex;
-using DistTestCore.Metrics;
 
 namespace CodexNetDeployer
 {
     public class Configuration
     {
         public const int SecondsIn1Day = 24 * 60 * 60;
-        public const int TenMinutes = 10 * 60;
 
         [Uniform("kube-config", "kc", "KUBECONFIG", false, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
         public string KubeConfigFile { get; set; } = "null";
@@ -45,22 +44,11 @@ namespace CodexNetDeployer
         [Uniform("block-ttl", "bt", "BLOCKTTL", false, "Block timeout in seconds. Default is 24 hours.")]
         public int BlockTTL { get; set; } = SecondsIn1Day;
 
-        [Uniform("block-mi", "bmi", "BLOCKMI", false, "Block maintenance interval in seconds. Default is 10 minutes.")]
-        public int BlockMI { get; set; } = TenMinutes;
-
-        [Uniform("block-mn", "bmn", "BLOCKMN", false, "Number of blocks maintained per interval. Default is 1000 blocks.")]
-        public int BlockMN { get; set; } = 1000;
-
-        [Uniform("metrics", "m", "METRICS", false, "[None*, Record, Dashboard]. Determines if metrics will be recorded and if a dashboard service will be created.")]
-        public MetricsMode Metrics { get; set; } = MetricsMode.None;
-
-        [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " +
-            "set this option to override the label value.")]
-        public string TestsTypePodLabel { get; set; } = "continuous-tests";
-
-        [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment.")]
-        public bool CheckPeerConnection { get; set; } = false;
+        [Uniform("record-metrics", "rm", "RECORDMETRICS", false, "If true, metrics will be collected for all Codex nodes.")]
+        public bool RecordMetrics { get; set; } = false;
        
+        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
+
         public List<string> Validate()
         {
             var errors = new List<string>();

CodexNetDeployer/Deployer.cs

@@ -8,29 +8,29 @@ namespace CodexNetDeployer
     public class Deployer
     {
         private readonly Configuration config;
+        private readonly NullLog log;
         private readonly DefaultTimeSet timeset;
-        private readonly PeerConnectivityChecker peerConnectivityChecker;
 
         public Deployer(Configuration config)
         {
             this.config = config;
+            log = new NullLog();
             timeset = new DefaultTimeSet();
-            peerConnectivityChecker = new PeerConnectivityChecker();
         }
 
         public CodexDeployment Deploy()
         {
             Log("Initializing...");
-            var lifecycle = CreateTestLifecycle();
+            var (workflowCreator, lifecycle) = CreateFacilities();
 
             Log("Preparing configuration...");
             // We trick the Geth companion node into unlocking all of its accounts, by saying we want to start 999 codex nodes.
             var setup = new CodexSetup(999, config.CodexLogLevel);
             setup.WithStorageQuota(config.StorageQuota!.Value.MB()).EnableMarketplace(0.TestTokens());
-            setup.MetricsMode = config.Metrics;
+            setup.MetricsEnabled = config.RecordMetrics;
 
             Log("Creating Geth instance and deploying contracts...");
-            var gethStarter = new GethStarter(lifecycle);
+            var gethStarter = new GethStarter(lifecycle, workflowCreator);
             var gethResults = gethStarter.BringOnlineMarketplaceFor(setup);
 
             Log("Geth started. Codex contracts deployed.");
@@ -44,23 +44,20 @@ namespace CodexNetDeployer
             Log("Starting Codex nodes...");
 
             // Each node must have its own IP, so it needs it own pod. Start them 1 at a time.
-            var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value);
-            var startResults = new List<CodexNodeStartResult>();
+            var codexStarter = new CodexNodeStarter(config, workflowCreator, lifecycle, gethResults, config.NumberOfValidators!.Value);
+            var codexContainers = new List<RunningContainer>();
             for (var i = 0; i < config.NumberOfCodexNodes; i++)
             {
-                var result = codexStarter.Start(i);
-                if (result != null) startResults.Add(result);
+                var container = codexStarter.Start(i);
+                if (container != null) codexContainers.Add(container);
             }
 
-            var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container));
+            var prometheusContainer = StartMetricsService(lifecycle, setup, codexContainers);
 
-            CheckPeerConnectivity(startResults);
-            CheckContainerRestarts(startResults);
-
-            return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata());
+            return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, CreateMetadata());
         }
 
-        private TestLifecycle CreateTestLifecycle()
+        private (WorkflowCreator, TestLifecycle) CreateFacilities()
         {
             var kubeConfig = GetKubeConfig(config.KubeConfigFile);
 
@@ -71,28 +68,28 @@ namespace CodexNetDeployer
                 logDebug: false,
                 dataFilesPath: "notUsed",
                 codexLogLevel: config.CodexLogLevel,
-                k8sNamespacePrefix: config.KubeNamespace
+                runnerLocation: config.RunnerLocation
             );
 
-            var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, timeset, string.Empty);
-            DefaultContainerRecipe.TestsType = config.TestsTypePodLabel;
-            DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
-            return lifecycle;
+            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
+                k8sNamespacePrefix: config.KubeNamespace,
+                kubeConfigFile: kubeConfig,
+                operationTimeout: timeset.K8sOperationTimeout(),
+                retryDelay: timeset.WaitForK8sServiceDelay());
+
+            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
+            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeset, workflowCreator);
+
+            return (workflowCreator, lifecycle);
         }
 
-        private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable<RunningContainer> codexContainers)
+        private RunningContainer? StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List<RunningContainer> codexContainers)
         {
-            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null);
+            if (!setup.MetricsEnabled) return null;
 
             Log("Starting metrics service...");
-            var runningContainers = new[] { new RunningContainers(null!, null!, codexContainers.ToArray()) };
-            var prometheusContainer = lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
-
-            if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.Record) return (prometheusContainer, null);
-
-            Log("Starting dashboard service...");
-            var grafanaStartInfo = lifecycle.GrafanaStarter.StartDashboard(prometheusContainer, setup);
-            return (prometheusContainer, grafanaStartInfo);
+            var runningContainers = new RunningContainers(null!, null!, codexContainers.ToArray());
+            return lifecycle.PrometheusStarter.CollectMetricsFor(runningContainers).Containers.Single();
         }
 
         private string? GetKubeConfig(string kubeConfigFile)
@@ -101,35 +98,6 @@ namespace CodexNetDeployer
             return kubeConfigFile;
         }
 
-        private void CheckPeerConnectivity(List<CodexNodeStartResult> codexContainers)
-        {
-            if (!config.CheckPeerConnection) return;
-
-            Log("Starting peer-connectivity check for deployed nodes...");
-            peerConnectivityChecker.CheckConnectivity(codexContainers);
-            Log("Check passed.");
-        }
-
-        private void CheckContainerRestarts(List<CodexNodeStartResult> startResults)
-        {
-            var crashes = new List<RunningContainer>();
-            foreach (var startResult in startResults)
-            {
-                var watcher = startResult.Workflow.CreateCrashWatcher(startResult.Container);
-                if (watcher.HasContainerCrashed()) crashes.Add(startResult.Container);
-            }
-
-            if (!crashes.Any())
-            {
-                Log("Container restart check passed.");
-            }
-            else
-            {
-                Log($"Deployment failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}");
-                throw new Exception("Deployment failed: One or more containers crashed.");
-            }
-        }
-
         private DeploymentMetadata CreateMetadata()
         {
             return new DeploymentMetadata(
@@ -141,10 +109,7 @@ namespace CodexNetDeployer
                 initialTestTokens: config.InitialTestTokens,
                 minPrice: config.MinPrice,
                 maxCollateral: config.MaxCollateral,
-                maxDuration: config.MaxDuration,
-                blockTTL: config.BlockTTL,
-                blockMI: config.BlockMI,
-                blockMN: config.BlockMN);
+                maxDuration: config.MaxDuration);
         }
 
         private void Log(string msg)

CodexNetDeployer/PeerConnectivityChecker.cs

@@ -1,34 +0,0 @@
-using DistTestCore.Helpers;
-using Logging;
-
-namespace CodexNetDeployer
-{
-    public class PeerConnectivityChecker
-    {
-        public void CheckConnectivity(List<CodexNodeStartResult> startResults)
-        {
-            var log = new ConsoleLog();
-            var checker = new PeerConnectionTestHelpers(log);
-            var access = startResults.Select(r => r.Access);
-
-            checker.AssertFullyConnected(access);
-        }
-    }
-
-    public class ConsoleLog : BaseLog
-    {
-        public ConsoleLog() : base(false)
-        {
-        }
-
-        protected override string GetFullName()
-        {
-            return "CONSOLE";
-        }
-
-        public override void Log(string message)
-        {
-            Console.WriteLine(message);
-        }
-    }
-}

CodexNetDeployer/Program.cs

@@ -1,5 +1,6 @@
 using ArgsUniform;
 using CodexNetDeployer;
+using DistTestCore;
 using DistTestCore.Codex;
 using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
@@ -16,6 +17,11 @@ public class Program
         var uniformArgs = new ArgsUniform<Configuration>(PrintHelp, args);
         var config = uniformArgs.Parse(true);
         
+        if (args.Any(a => a == "--external"))
+        {
+            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
+        }
+
         var errors = config.Validate();
         if (errors.Any())
         {
@@ -27,11 +33,10 @@ public class Program
         }
 
         Console.WriteLine("Using images:" + nl +
-            $"\tCodex image: '{new CodexContainerRecipe().Image}'" + nl +
-            $"\tCodexContracts image: '{new CodexContractsContainerRecipe().Image}'" + nl +
-            $"\tPrometheus image: '{new PrometheusContainerRecipe().Image}'" + nl +
-            $"\tGeth image: '{new GethContainerRecipe().Image}'" + nl +
-            $"\tGrafana image: '{new GrafanaContainerRecipe().Image}'" + nl);
+            $"\tCodex image: '{CodexContainerRecipe.DockerImage}'" + nl +
+            $"\tCodex Contracts image: '{CodexContractsContainerRecipe.DockerImage}'" + nl +
+            $"\tPrometheus image: '{PrometheusContainerRecipe.DockerImage}'" + nl +
+            $"\tGeth image: '{GethContainerRecipe.DockerImage}'" + nl);
 
         if (!args.Any(a => a == "-y"))
         {
@@ -56,5 +61,8 @@ public class Program
         Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " +
             "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " +
             "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl);
+
+        Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " +
+            "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
     }
 }

CodexNetDeployer/deploy-continuous-testnet.sh

@@ -3,14 +3,10 @@ dotnet run \
     --kube-namespace=codex-continuous-tests \
     --nodes=5 \
     --validators=3 \
-    --log-level=Trace \
     --storage-quota=2048 \
     --storage-sell=1024 \
     --min-price=1024 \
     --max-collateral=1024 \
     --max-duration=3600000 \
-    --block-ttl=180 \
-    --block-mi=120 \
-    --block-mn=10000 \
-    --metrics=Dashboard \
-    --check-connect=1
+    --block-ttl=120
+

CodexNetDownloader/Configuration.cs

@@ -1,4 +1,5 @@
 using ArgsUniform;
+using DistTestCore;
 using DistTestCore.Codex;
 
 namespace CodexNetDownloader
@@ -15,5 +16,7 @@ namespace CodexNetDownloader
         public string KubeConfigFile { get; set; } = "null";
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
+
+        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
     }
 }

CodexNetDownloader/Program.cs

@@ -15,12 +15,17 @@ public class Program
         var uniformArgs = new ArgsUniform<CodexNetDownloader.Configuration>(PrintHelp, args);
         var config = uniformArgs.Parse(true);
 
+        if (args.Any(a => a == "--external"))
+        {
+            config.RunnerLocation = TestRunnerLocation.ExternalToCluster;
+        }
+
         config.CodexDeployment = ParseCodexDeploymentJson(config.CodexDeploymentJson);
 
         if (!Directory.Exists(config.OutputPath)) Directory.CreateDirectory(config.OutputPath);
 
         var k8sFactory = new K8sFactory();
-        var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog());
+        var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.OutputPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
 
         foreach (var container in config.CodexDeployment.CodexContainers)
         {

ContinuousTests/CodexAccessFactory.cs

@@ -12,7 +12,7 @@ namespace ContinuousTests
             return containers.Select(container =>
             {
                 var address = container.ClusterExternalAddress;
-                if (config.RunnerLocation == RunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
+                if (config.RunnerLocation == TestRunnerLocation.InternalToCluster) address = container.ClusterInternalAddress;
                 return new CodexAccess(log, container, timeSet, address);
             }).ToArray();
         }

ContinuousTests/Configuration.cs

@@ -22,15 +22,12 @@ namespace ContinuousTests
         [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
         public string KubeConfigFile { get; set; } = "null";
 
-        [Uniform("stop", "s", "STOPONFAIL", false, "If greater than zero, runner will stop after this many test failures and download all cluster container logs. 0 by default.")]
-        public int StopOnFailure { get; set; } = 0;
-
-        [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
-        public bool DownloadContainerLogs { get; set; } = false;
+        [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
+        public bool StopOnFailure { get; set; } = false;
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
 
-        public RunnerLocation RunnerLocation { get; set; }
+        public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
     }
 
     public class ConfigLoader
@@ -42,7 +39,10 @@ namespace ContinuousTests
             var result = uniformArgs.Parse(true);
             
             result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
-            result.RunnerLocation = RunnerLocationUtils.DetermineRunnerLocation(result.CodexDeployment.CodexContainers.First());
+            if (args.Any(a => a == "--external"))
+            {
+                result.RunnerLocation = TestRunnerLocation.ExternalToCluster;
+            }
 
             return result;
         }
@@ -57,10 +57,9 @@ namespace ContinuousTests
         private static void PrintHelp()
         {
             var nl = Environment.NewLine;
-            Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
-                "The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
+            Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
 
-            Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
+            Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
                 "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
         }
     }

ContinuousTests/ContinuousTest.cs

@@ -1,7 +1,5 @@
 using DistTestCore;
 using DistTestCore.Codex;
-using DistTestCore.Logs;
-using KubernetesWorkflow;
 using Logging;
 
 namespace ContinuousTests
@@ -89,12 +87,6 @@ namespace ContinuousTests
             return file;
         }
 
-        public IDownloadedLog DownloadContainerLog(RunningContainer container, int? tailLines = null)
-        {
-            var nodeRunner = new NodeRunner(Nodes, Configuration, TimeSet, Log, Configuration.CodexDeployment.Metadata.KubeNamespace, EthereumAccountIndex);
-            return nodeRunner.DownloadLog(container, tailLines);
-        }
-
         private void DownloadToFile(CodexAccess node, string contentId, TestFile file)
         {
             using var fileStream = File.OpenWrite(file.Filename);

ContinuousTests/ContinuousTestRunner.cs

@@ -24,13 +24,13 @@ namespace ContinuousTests
             startupChecker.Check();
 
             var taskFactory = new TaskFactory();
-            var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "Overview");
+            var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
             overviewLog.Log("Continuous tests starting...");
             var allTests = testFactory.CreateTests();
 
             ClearAllCustomNamespaces(allTests, overviewLog);
 
-            var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
+            var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
 
             foreach (var testLoop in testLoops)
             {
@@ -58,8 +58,8 @@ namespace ContinuousTests
             if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
 
             log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
-            var lifecycle = k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log);
-            lifecycle.WorkflowCreator.CreateWorkflow().DeleteTestResources();
+            var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
+            workflowCreator.CreateWorkflow().DeleteTestResources();
         }
     }
 }

ContinuousTests/K8sFactory.cs

@@ -1,12 +1,13 @@
 using DistTestCore.Codex;
 using DistTestCore;
+using KubernetesWorkflow;
 using Logging;
 
 namespace ContinuousTests
 {
     public class K8sFactory
     {
-        public TestLifecycle CreateTestLifecycle(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log)
+        public (WorkflowCreator, TestLifecycle) CreateFacilities(string kubeConfigFile, string logPath, string dataFilePath, string customNamespace, ITimeSet timeSet, BaseLog log, TestRunnerLocation runnerLocation)
         {
             var kubeConfig = GetKubeConfig(kubeConfigFile);
             var lifecycleConfig = new DistTestCore.Configuration
@@ -16,13 +17,19 @@ namespace ContinuousTests
                 logDebug: false,
                 dataFilesPath: dataFilePath,
                 codexLogLevel: CodexLogLevel.Debug,
-                k8sNamespacePrefix: customNamespace
+                runnerLocation: runnerLocation
             );
 
-            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, string.Empty);
-            DefaultContainerRecipe.TestsType = "continuous-tests";
-            DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
-            return lifecycle;
+            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
+                k8sNamespacePrefix: customNamespace,
+                kubeConfigFile: kubeConfig,
+                operationTimeout: timeSet.K8sOperationTimeout(),
+            retryDelay: timeSet.WaitForK8sServiceDelay());
+
+            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
+            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator);
+
+            return (workflowCreator, lifecycle);
         }
 
         private static string? GetKubeConfig(string kubeConfigFile)

ContinuousTests/NodeRunner.cs

@@ -5,7 +5,6 @@ using KubernetesWorkflow;
 using NUnit.Framework;
 using Logging;
 using Utils;
-using DistTestCore.Logs;
 
 namespace ContinuousTests
 {
@@ -39,25 +38,10 @@ namespace ContinuousTests
             RunNode(bootstrapNode, operation, 0.TestTokens());
         }
 
-        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
-        {
-            var subFile = log.CreateSubfile();
-            var description = container.Name;
-            var handler = new LogDownloadHandler(container, description, subFile);
-
-            log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-
-            var lifecycle = CreateTestLifecycle();
-            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
-            flow.DownloadContainerLog(container, handler, tailLines);
-
-            return new DownloadedLog(subFile, description);
-        }
-
         public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess, TestLifecycle> operation, TestToken mintTestTokens)
         {
-            var lifecycle = CreateTestLifecycle();
-            var flow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var (workflowCreator, lifecycle) = CreateFacilities();
+            var flow = workflowCreator.CreateWorkflow();
 
             try
             {
@@ -105,9 +89,9 @@ namespace ContinuousTests
             }
         }
 
-        private TestLifecycle CreateTestLifecycle()
+        private (WorkflowCreator, TestLifecycle) CreateFacilities()
         {
-            return k8SFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log);
+            return k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, customNamespace, timeSet, log, config.RunnerLocation);
         }
     }
 }

ContinuousTests/SingleTestRun.cs

@@ -23,9 +23,8 @@ namespace ContinuousTests
         private readonly FixtureLog fixtureLog;
         private readonly string testName;
         private readonly string dataFolder;
-        private static int failureCount = 0;
 
-        public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken)
+        public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, CancellationToken cancelToken)
         {
             this.taskFactory = taskFactory;
             this.config = config;
@@ -33,10 +32,9 @@ namespace ContinuousTests
             this.handle = handle;
             this.cancelToken = cancelToken;
             testName = handle.Test.GetType().Name;
-            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
-            ApplyLogReplacements(fixtureLog, startupChecker);
+            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), testName);
 
-            nodes = CreateRandomNodes();
+            nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
             dataFolder = config.DataPath + "-" + Guid.NewGuid();
             fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration());
         }
@@ -73,26 +71,16 @@ namespace ContinuousTests
                 fixtureLog.Error("Test run failed with exception: " + ex);
                 fixtureLog.MarkAsFailed();
 
-                failureCount++;
-                if (config.StopOnFailure > 0)
+                if (config.StopOnFailure)
                 {
-                    OverviewLog($"Failures: {failureCount} / {config.StopOnFailure}");
-                    if (failureCount >= config.StopOnFailure)
-                    {
-                        OverviewLog($"Configured to stop after {config.StopOnFailure} failures. Downloading cluster logs...");
-                        DownloadClusterLogs();
-                        OverviewLog("Log download finished. Cancelling test runner...");
-                        Cancellation.Cts.Cancel();
-                    }
+                    OverviewLog("Configured to stop on first failure. Downloading cluster logs...");
+                    DownloadClusterLogs();
+                    OverviewLog("Log download finished. Cancelling test runner...");
+                    Cancellation.Cts.Cancel();
                 }
             }
         }
 
-        private void ApplyLogReplacements(FixtureLog fixtureLog, StartupChecker startupChecker)
-        {
-            foreach (var replacement in startupChecker.LogReplacements) fixtureLog.AddStringReplace(replacement.From, replacement.To);
-        }
-
         private void RunTestMoments()
         {
             var earliestMoment = handle.GetEarliestMoment();
@@ -124,7 +112,7 @@ namespace ContinuousTests
                     {
                         ThrowFailTest();
                     }
-                    OverviewLog(" > Test passed.");
+                    OverviewLog(" > Test passed. " + FuturesInfo());
                     return;
                 }
             }
@@ -132,19 +120,25 @@ namespace ContinuousTests
 
         private void ThrowFailTest()
         {
-            var exs = UnpackExceptions(exceptions);
-            var exceptionsMessage = GetCombinedExceptionsMessage(exs);
-            Log(exceptionsMessage);
-            OverviewLog($" > Test failed: " + exceptionsMessage);
-            throw new Exception(exceptionsMessage);
+            var ex = UnpackException(exceptions.First());
+            Log(ex.ToString());
+            OverviewLog($" > Test failed {FuturesInfo()}: " + ex.Message);
+            throw ex;
+        }
+
+        private string FuturesInfo()
+        {
+            var containers = config.CodexDeployment.CodexContainers;
+            var nodes = codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
+            var f = nodes.Select(n => n.GetDebugFutures().ToString());
+            var msg = $"(Futures: [{string.Join(", ", f)}])";
+            return msg;
         }
 
         private void DownloadClusterLogs()
         {
             var k8sFactory = new K8sFactory();
-            var log = new NullLog();
-            log.FullFilename = Path.Combine(config.LogPath, "NODE");
-            var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), log);
+            var (_, lifecycle) = k8sFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
 
             foreach (var container in config.CodexDeployment.CodexContainers)
             {
@@ -152,16 +146,6 @@ namespace ContinuousTests
             }
         }
 
-        private string GetCombinedExceptionsMessage(Exception[] exceptions)
-        {
-            return string.Join(Environment.NewLine, exceptions.Select(ex => ex.ToString()));
-        }
-
-        private Exception[] UnpackExceptions(List<Exception> exceptions)
-        {
-            return exceptions.Select(UnpackException).ToArray();
-        }
-
         private Exception UnpackException(Exception exception)
         {
             if (exception is AggregateException a)
@@ -212,28 +196,19 @@ namespace ContinuousTests
         private void OverviewLog(string msg)
         {
             Log(msg);
-            var containerNames = GetContainerNames();
+            var containerNames = $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
             overviewLog.Log($"{containerNames} {testName}: {msg}");
         }
 
-        private string GetContainerNames()
+        private CodexAccess[] CreateRandomNodes(int number)
         {
-            if (handle.Test.RequiredNumberOfNodes == -1) return "(All Nodes)";
-            return $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
-        }
-
-        private CodexAccess[] CreateRandomNodes()
-        {
-            var containers = SelectRandomContainers();
+            var containers = SelectRandomContainers(number);
             fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
             return codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
         }
 
-        private RunningContainer[] SelectRandomContainers()
+        private RunningContainer[] SelectRandomContainers(int number)
         {
-            var number = handle.Test.RequiredNumberOfNodes;
-            if (number == -1) return config.CodexDeployment.CodexContainers;
-
             var containers = config.CodexDeployment.CodexContainers.ToList();
             var result = new RunningContainer[number];
             for (var i = 0; i < number; i++)
@@ -246,7 +221,7 @@ namespace ContinuousTests
         private DistTestCore.Configuration CreateFileManagerConfiguration()
         {
             return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
-                CodexLogLevel.Error, string.Empty);
+                CodexLogLevel.Error, config.RunnerLocation);
         }
     }
 }

ContinuousTests/StartupChecker.cs

@@ -15,12 +15,11 @@ namespace ContinuousTests
         {
             this.config = config;
             this.cancelToken = cancelToken;
-            LogReplacements = new List<BaseLogStringReplacement>();
         }
 
         public void Check()
         {
-            var log = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "StartupChecks");
+            var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
             log.Log("Starting continuous test run...");
             log.Log("Checking configuration...");
             PreflightCheck(config);
@@ -29,8 +28,6 @@ namespace ContinuousTests
             log.Log("All OK.");
         }
 
-        public List<BaseLogStringReplacement> LogReplacements { get; }
-
         private void PreflightCheck(Configuration config)
         {
             var tests = testFactory.CreateTests();
@@ -93,7 +90,6 @@ namespace ContinuousTests
                 if (info == null || string.IsNullOrEmpty(info.id)) return false;
 
                 log.Log($"Codex version: '{info.codex.version}' revision: '{info.codex.revision}'");
-                LogReplacements.Add(new BaseLogStringReplacement(info.id, n.GetName()));
             }
             catch
             {
@@ -160,16 +156,9 @@ namespace ContinuousTests
         {
             foreach (var test in tests)
             {
-                if (test.RequiredNumberOfNodes != -1)
+                if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
                 {
-                    if (test.RequiredNumberOfNodes < 1)
-                    {
-                        errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Test must require > 0 nodes, or -1 to select all nodes.");
-                    }
-                    else if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexContainers.Length)
-                    {
-                        errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
-                    }
+                    errors.Add($"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexContainers.Length}");
                 }
             }
         }

ContinuousTests/TestLoop.cs

@@ -9,18 +9,16 @@ namespace ContinuousTests
         private readonly BaseLog overviewLog;
         private readonly Type testType;
         private readonly TimeSpan runsEvery;
-        private readonly StartupChecker startupChecker;
         private readonly CancellationToken cancelToken;
         private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset);
 
-        public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken)
+        public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, CancellationToken cancelToken)
         {
             this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
             this.testType = testType;
             this.runsEvery = runsEvery;
-            this.startupChecker = startupChecker;
             this.cancelToken = cancelToken;
             Name = testType.Name;
         }
@@ -60,7 +58,7 @@ namespace ContinuousTests
         {
             var test = (ContinuousTest)Activator.CreateInstance(testType)!;
             var handle = new TestHandle(test);
-            var run = new SingleTestRun(taskFactory, config, overviewLog, handle, startupChecker, cancelToken);
+            var run = new SingleTestRun(taskFactory, config, overviewLog, handle, cancelToken);
 
             runFinishedHandle.Reset();
             run.Run(runFinishedHandle);

ContinuousTests/Tests/FetchTest.cs

@@ -1,18 +0,0 @@
-using DistTestCore.Helpers;
-
-namespace ContinuousTests.Tests
-{
-    public class FetchTest : ContinuousTest
-    {
-        public override int RequiredNumberOfNodes => -1;
-        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);
-        public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
-
-        [TestMoment(t: 0)]
-        public void CheckConnectivity()
-        {
-            var checker = new PeerFetchTestHelpers(Log, FileManager);
-            checker.AssertFullFetchInterconnectivity(Nodes);
-        }
-    }
-}

ContinuousTests/Tests/HoldMyBeerTest.cs

@@ -1,29 +0,0 @@
-using DistTestCore;
-using NUnit.Framework;
-
-namespace ContinuousTests.Tests
-{
-    public class HoldMyBeerTest : ContinuousTest
-    {
-        public override int RequiredNumberOfNodes => 1;
-        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);
-        public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
-
-        private ContentId? cid;
-        private TestFile file = null!;
-
-        [TestMoment(t: Zero)]
-        public void UploadTestFile()
-        {
-            var filesize = 80.MB();
-
-            file = FileManager.GenerateTestFile(filesize);
-
-            cid = UploadFile(Nodes[0], file);
-            Assert.That(cid, Is.Not.Null);
-
-            var dl = DownloadFile(Nodes[0], cid!);
-            file.AssertIsEqual(dl);
-        }
-    }
-}

ContinuousTests/Tests/MarketplaceTest.cs

@@ -60,5 +60,40 @@
 //                file.AssertIsEqual(result);
 //            });
 //        }
+
+//        private void WaitForContractToStart(CodexAccess codexAccess, string purchaseId)
+//        {
+//            var lastState = "";
+//            var waitStart = DateTime.UtcNow;
+//            var filesizeInMb = fileSize.SizeInBytes / (1024 * 1024);
+//            var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0);
+
+//            Log.Log($"{nameof(WaitForContractToStart)} for {Time.FormatDuration(maxWaitTime)}");
+//            while (lastState != "started")
+//            {
+//                CancelToken.ThrowIfCancellationRequested();
+
+//                var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId);
+//                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
+//                if (purchaseStatus != null && purchaseStatus.state != lastState)
+//                {
+//                    lastState = purchaseStatus.state;
+//                    Log.Log("Purchase status: " + statusJson);
+//                }
+
+//                Thread.Sleep(2000);
+
+//                if (lastState == "errored")
+//                {
+//                    Assert.Fail("Contract start failed: " + statusJson);
+//                }
+
+//                if (DateTime.UtcNow - waitStart > maxWaitTime)
+//                {
+//                    Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: {statusJson}");
+//                }
+//            }
+//            Log.Log("Contract started.");
+//        }
 //    }
 //}

ContinuousTests/Tests/PeersTest.cs

@@ -1,57 +0,0 @@
-using DistTestCore.Codex;
-using DistTestCore.Helpers;
-using NUnit.Framework;
-
-namespace ContinuousTests.Tests
-{
-    public class PeersTest : ContinuousTest
-    {
-        public override int RequiredNumberOfNodes => -1;
-        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);
-        public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
-
-        [TestMoment(t: 0)]
-        public void CheckConnectivity()
-        {
-            var checker = new PeerConnectionTestHelpers(Log);
-            checker.AssertFullyConnected(Nodes);
-        }
-
-        [TestMoment(t: 10)]
-        public void CheckRoutingTables()
-        {
-            var allInfos = Nodes.Select(n =>
-            {
-                var info = n.GetDebugInfo();
-                Log.Log($"{n.GetName()} = {info.table.localNode.nodeId}");
-                Log.AddStringReplace(info.table.localNode.nodeId, n.GetName());
-                return info;
-            }).ToArray();
-
-            var allIds = allInfos.Select(i => i.table.localNode.nodeId).ToArray();
-            var errors = Nodes.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray();
-
-            if (errors.Any())
-            {
-                Assert.Fail(string.Join(Environment.NewLine, errors));
-            }
-        }
-
-        private string AreAllPresent(CodexAccess n, string[] allIds)
-        {
-            var info = n.GetDebugInfo();
-            var known = info.table.nodes.Select(n => n.nodeId).ToArray();
-            var expected = allIds.Where(i => i != info.table.localNode.nodeId).ToArray();
-
-            if (!expected.All(ex => known.Contains(ex)))
-            {
-                var nl = Environment.NewLine;
-                return $"{nl}At node '{info.table.localNode.nodeId}'{nl}" +
-                    $"Not all of{nl}'{string.Join(",", expected)}'{nl}" +
-                    $"were present in routing table:{nl}'{string.Join(",", known)}'";
-            }
-
-            return string.Empty;
-        }
-    }
-}

ContinuousTests/Tests/TwoClientTest.cs

@@ -6,7 +6,7 @@ namespace ContinuousTests.Tests
     public class TwoClientTest : ContinuousTest
     {
         public override int RequiredNumberOfNodes => 2;
-        public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(2);
+        public override TimeSpan RunTestEvery => TimeSpan.FromSeconds(30);
         public override TestFailMode TestFailMode => TestFailMode.StopAfterFirstFailure;
 
         private ContentId? cid;
@@ -15,7 +15,7 @@ namespace ContinuousTests.Tests
         [TestMoment(t: Zero)]
         public void UploadTestFile()
         {
-            file = FileManager.GenerateTestFile(80.MB());
+            file = FileManager.GenerateTestFile(10.MB());
 
             cid = UploadFile(Nodes[0], file);
             Assert.That(cid, Is.Not.Null);

ContinuousTests/reports/CodexTestNetReport-August2023.md

@@ -1,49 +0,0 @@
-# Codex Continuous Test-net Report
-Date: 05-09-2023
-
-Report for: 08-2023
-
-
-## Test-net Status
-- Start of month: Offline - faulted
-- End of month: Offline - stopped
-
-(Faulted: Tests fail with such frequency that the information gathered does not justify the cost of leaving the test-net running.)
-
-(Stopped: The number of tests that can successfully run on the test-net is not high enough to justify the cost of leaving it running.)
-
-## Deployment Configuration
-Continous Test-net is deployed to the kubernetes cluster with the following configuration:
-
-5x Codex Nodes:
-- Log-level: Trace
-- Storage quota: 2048 MB
-- Storage sell: 1024 MB
-- Min price: 1024
-- Max collateral: 1024
-- Max duration: 3600000 seconds
-- Block-TTL: 180 seconds
-- Block-MI: 120 seconds
-- Block-MN: 10000 blocks
-
-3 of these 5 nodes have:
-- Validator: true
-
-Kubernetes namespace: 'codex-continuous-tests'
-
-## Test Overview
-| Changes        | Test                | Description                                                                                                                                                                                                                                                   | Status      | Results                                                                               |
-|----------------|---------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|---------------------------------------------------------------------------------------|
-| No change      | Two-client test     | See report for July 2023.                                                                                                                                                                                                                                     | Faulted     | Test reliably fails. Both upload and download failures occur.                         |
-| No change      | Transient-node test | See report for July 2023.                                                                                                                                                                                                                                     | Not running | Test was not run because the previous more rudamentory test has faulted.              |
-| New in 08-2023 | HoldMyBeer test     | Named so because the test is quite dumb and only holds a file, this test selects 1 node at random and uploads an 80 MB file. Then it downloads and verifies it. The purpose of this test is to isolate the upload/download behavior to identify issues in it. | Passed      | Several successful runs. Record run: 48h. After that the test was manually stopped.   |
-| New in 08-2023 | Peers test          | Another attempt to isolate a portion of the two-client test behavior. This test periodically checks the connectivity status between each node in the network. It does this by analysing routing-table information and by performing a node-find operation.    | Passed      | Several successful runs. Record run: 2d21h. After that the test was manually stopped. |
-
-## Resulting changes
-As a result of the testing efforts in 08-2023, these three changes were made:
-1. Block announcement performance - Block iteration (as part of the advertising loop) was found to be performing too much disc IO. Reducing this greatly increased client stability.
-1. Component initialization order - A timing issue in the initialization order of the several Codex components caused nodes to improperly announce their addresses over the network in case the marketplace support was enabled.
-1. Block maintenance performance - Block iteration (again but this time in maintenance) was causing the node to not respond to Ping messages quickly enough, causing other nodes to kick it from the network. By slowing down block maintenance this is solved. A future update of the DHT logic will also impact this behavior.
-
-## Action Points
-- Despite both HoldMyBeer and Peers test success, the two-client test is still not passing. In follow-up actions, we can investigate this issue further by decoupling block retrieval from file download.

ContinuousTests/reports/CodexTestNetReport-July2023.md

@@ -1,38 +0,0 @@
-# Codex Continuous Test-net Report
-Date: 02-08-2023
-
-Report for: 07-2023
-
-
-## Test-net Status
-- Start of month: Offline - faulted
-- End of month: Offline - faulted
-
-(Faulted: Tests fail with such frequency that the information gathered does not justify the cost of leaving the test-net running.)
-
-## Deployment Configuration
-Continous Test-net is deployed to the kubernetes cluster with the following configuration:
-
-5x Codex Nodes:
-- Log-level: Trace
-- Storage quota: 2048 MB
-- Storage sell: 1024 MB
-- Min price: 1024
-- Max collateral: 1024
-- Max duration: 3600000 seconds
-- Block-TTL: 120 seconds
-
-3 of these 5 nodes have:
-- Validator: true
-
-Kubernetes namespace: 'codex-continuous-tests'
-
-## Test Overview
-| Changes        | Test                | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | Status      | Results                                                                              |
-|----------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|--------------------------------------------------------------------------------------|
-| New in 07-2023 | Two-client test     | Every 30 seconds, two nodes are chosen at random. A 10 MB file is generated and uploaded to one. 10 seconds later, it is downloaded from the other. File contents are asserted to be equal.                                                                                                                                                                                                                                                                                                                                                                                    | Faulted     | Test reliably fails after 30 to 45 minutes. Both upload and download failures occur. |
-| New in 07-2023 | Transient-node test | Every 1 minute, a new, transient Codex node is started and bootstrapped against a random node of the test net. A 10 MB file is generated and uploaded to the transient node. The file is then downloaded from a second random test net node and file equality is asserted. After that, the transient node is shut down. 30 seconds later, a new transient Codex node is started and bootstrapped against a third (guaranteed different from first and second) node from the test net. The same file is then downloaded from the new transient node. File equality is asserted. | Not running | Test was not run because the previous more rudamentory test has faulted.             |
-
-## Action Points
-- Codex logs (from these long-running containers) are often incomplete when downloaded after a test failure. A reliable way of maintaining these logs is needed. The logs can quickly explode in size, even for test-nets that run only for a few hours.
-- The distributed testing setup can and has been used to reproduce the failure of the Two-client test in a local environment. Investigation is on-going.

ContinuousTests/run-continuous-tests.sh

@@ -1,6 +1,4 @@
 dotnet run \
     --kube-config=/opt/kubeconfig.yaml \
     --codex-deployment=codex-deployment.json \
-    --keep=1 \
-    --stop=10 \
-    --dl-logs=1
+    --stop=1 

DistTestCore/BaseStarter.cs

@@ -1,15 +1,18 @@
-using Logging;
+using KubernetesWorkflow;
+using Logging;
 
 namespace DistTestCore
 {
     public class BaseStarter
     {
         protected readonly TestLifecycle lifecycle;
+        protected readonly WorkflowCreator workflowCreator;
         private Stopwatch? stopwatch;
 
-        public BaseStarter(TestLifecycle lifecycle)
+        public BaseStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
         {
             this.lifecycle = lifecycle;
+            this.workflowCreator = workflowCreator;
         }
 
         protected void LogStart(string msg)

DistTestCore/ByteSize.cs

@@ -38,11 +38,6 @@ namespace DistTestCore
     {
         private const long Kilo = 1024;
 
-        public static ByteSize Bytes(this long i)
-        {
-            return new ByteSize(i);
-        }
-
         public static ByteSize KB(this long i)
         {
             return new ByteSize(i * Kilo);
@@ -63,11 +58,6 @@ namespace DistTestCore
             return (i * Kilo).GB();
         }
 
-        public static ByteSize Bytes(this int i)
-        {
-            return new ByteSize(i);
-        }
-
         public static ByteSize KB(this int i)
         {
             return Convert.ToInt64(i).KB();

DistTestCore/Codex/CodexAccess.cs

@@ -4,11 +4,10 @@ using Utils;
 
 namespace DistTestCore.Codex
 {
-    public class CodexAccess : ILogHandler
+    public class CodexAccess
     {
         private readonly BaseLog log;
         private readonly ITimeSet timeSet;
-        private bool hasContainerCrashed;
 
         public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
         {
@@ -16,9 +15,6 @@ namespace DistTestCore.Codex
             Container = container;
             this.timeSet = timeSet;
             Address = address;
-            hasContainerCrashed = false;
-
-            if (container.CrashWatcher != null) container.CrashWatcher.Start(this);
         }
 
         public RunningContainer Container { get; }
@@ -47,10 +43,10 @@ namespace DistTestCore.Codex
             return result;
         }
 
-        public CodexDebugFetchResponse DebugFetch(string contentId)
+        public int GetDebugFutures()
         {
-            var http = Http();
-            return http.HttpGetJson<CodexDebugFetchResponse>($"debug/fetch/{contentId}");
+            // Some Codex images support debug/futures to count the number of open futures.
+            return 0; // Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
         }
 
         public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
@@ -88,36 +84,9 @@ namespace DistTestCore.Codex
             return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
         }
 
-        public string GetName()
-        {
-            return Container.Name;
-        }
-
         private Http Http()
         {
-            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
-        }
-
-        private void CheckContainerCrashed(HttpClient client)
-        {
-            if (hasContainerCrashed) throw new Exception("Container has crashed.");
-        }
-
-        public void Log(Stream crashLog)
-        {
-            var file = log.CreateSubfile();
-            log.Log($"Container {Container.Name} has crashed. Downloading crash log to '{file.FullFilename}'...");
-
-            using var reader = new StreamReader(crashLog);
-            var line = reader.ReadLine();
-            while (line != null)
-            {
-                file.Write(line);
-                line = reader.ReadLine();
-            }
-
-            log.Log("Crash log successfully downloaded.");
-            hasContainerCrashed = true;
+            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name);
         }
     }
 }

DistTestCore/Codex/CodexApiTypes.cs

@@ -1,4 +1,6 @@
-using Newtonsoft.Json;
+using KubernetesWorkflow;
+using Logging;
+using Utils;
 
 namespace DistTestCore.Codex
 {
@@ -58,16 +60,6 @@ namespace DistTestCore.Codex
     {
         public string version { get; set; } = string.Empty;
         public string revision { get; set; } = string.Empty;
-
-        public bool IsValid()
-        {
-            return !string.IsNullOrEmpty(version) && !string.IsNullOrEmpty(revision);
-        }
-
-        public override string ToString()
-        {
-            return JsonConvert.SerializeObject(this);
-        }
     }
 
     public class CodexDebugPeerResponse
@@ -84,23 +76,6 @@ namespace DistTestCore.Codex
         public string address { get; set; } = string.Empty;
     }
 
-    public class CodexDebugFetchResponse
-    {
-        public string originalBytes { get; set; } = string.Empty;
-        public string blockSize { get; set; } = string.Empty;
-        public string numberOfBlocks { get; set; } = string.Empty;
-        public string version { get; set; } = string.Empty;
-        public string hcodec { get; set; } = string.Empty;
-        public string codec { get; set; } = string.Empty;
-        public string @protected { get; set; } = string.Empty;
-        public CodexDebugFetchBlockResponse[] blocks { get; set; } = Array.Empty<CodexDebugFetchBlockResponse>();
-    }
-
-    public class CodexDebugFetchBlockResponse
-    {
-        public string cid { get; set; } = string.Empty;
-    }
-
     public class CodexDebugThresholdBreaches
     {
         public string[] breaches { get; set; } = Array.Empty<string>();

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -3,27 +3,33 @@ using KubernetesWorkflow;
 
 namespace DistTestCore.Codex
 {
-    public class CodexContainerRecipe : DefaultContainerRecipe
+    public class CodexContainerRecipe : ContainerRecipeFactory
     {
-        //private const string DefaultDockerImage = "codexstorage/nim-codex:sha-d279eeb-dist-tests";
-        private const string DefaultDockerImage = "thatbenbierens/nim-codex:debugfetch";
-
+#if Arm64
+        public const string DockerImage = "codexstorage/nim-codex:sha-7227a4a";
+#else
+        //public const string DockerImage = "thatbenbierens/nim-codex:loopingyeah";
+        public const string DockerImage = "codexstorage/nim-codex:sha-7227a4a";
+#endif
         public const string MetricsPortTag = "metrics_port";
         public const string DiscoveryPortTag = "discovery-port";
 
-        // Used by tests for time-constraint assertions.
+        // Used by tests for time-constraint assersions.
         public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
         public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
 
-        public override string AppName => "codex";
-        public override string Image { get; }
+        public static string DockerImageOverride = string.Empty;
 
-        public CodexContainerRecipe()
+        protected override string Image
         {
-            Image = GetDockerImage();
+            get
+            {
+                if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride;
+                return DockerImage;
+            }
         }
 
-        protected override void InitializeRecipe(StartupConfig startupConfig)
+        protected override void Initialize(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<CodexStartupConfig>();
 
@@ -52,22 +58,11 @@ namespace DistTestCore.Codex
             {
                 AddEnvVar("CODEX_BLOCK_TTL", config.BlockTTL.ToString()!);
             }
-            if (config.BlockMaintenanceInterval != null)
+            if (config.MetricsEnabled)
             {
-                AddEnvVar("CODEX_BLOCK_MI", Convert.ToInt32(config.BlockMaintenanceInterval.Value.TotalSeconds).ToString());
-            }
-            if (config.BlockMaintenanceNumber != null)
-            {
-                AddEnvVar("CODEX_BLOCK_MN", config.BlockMaintenanceNumber.ToString()!);
-            }
-            if (config.MetricsMode != Metrics.MetricsMode.None)
-            {
-                var metricsPort = AddInternalPort(MetricsPortTag);
                 AddEnvVar("CODEX_METRICS", "true");
                 AddEnvVar("CODEX_METRICS_ADDRESS", "0.0.0.0");
-                AddEnvVar("CODEX_METRICS_PORT", metricsPort);
-                AddPodAnnotation("prometheus.io/scrape", "true");
-                AddPodAnnotation("prometheus.io/port", metricsPort.Number.ToString());
+                AddInternalPortAndVar("CODEX_METRICS_PORT", tag: MetricsPortTag);
             }
 
             if (config.MarketplaceConfig != null)
@@ -97,12 +92,5 @@ namespace DistTestCore.Codex
             if (marketplaceConfig.AccountIndexOverride != null) return marketplaceConfig.AccountIndexOverride.Value;
             return Index;
         }
-
-        private string GetDockerImage()
-        {
-            var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
-            if (!string.IsNullOrEmpty(image)) return image;
-            return DefaultDockerImage;
-        }
     }
 }

DistTestCore/Codex/CodexDeployment.cs

@@ -5,25 +5,23 @@ namespace DistTestCore.Codex
 {
     public class CodexDeployment
     {
-        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, GrafanaStartInfo? grafanaStartInfo, DeploymentMetadata metadata)
+        public CodexDeployment(GethStartResult gethStartResult, RunningContainer[] codexContainers, RunningContainer? prometheusContainer, DeploymentMetadata metadata)
         {
             GethStartResult = gethStartResult;
             CodexContainers = codexContainers;
             PrometheusContainer = prometheusContainer;
-            GrafanaStartInfo = grafanaStartInfo;
             Metadata = metadata;
         }
 
         public GethStartResult GethStartResult { get; }
         public RunningContainer[] CodexContainers { get; }
         public RunningContainer? PrometheusContainer { get; }
-        public GrafanaStartInfo? GrafanaStartInfo { get; }
         public DeploymentMetadata Metadata { get; }
     }
 
     public class DeploymentMetadata
     {
-        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN)
+        public DeploymentMetadata(string kubeNamespace, int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration)
         {
             DeployDateTimeUtc = DateTime.UtcNow;
             KubeNamespace = kubeNamespace;
@@ -35,9 +33,6 @@ namespace DistTestCore.Codex
             MinPrice = minPrice;
             MaxCollateral = maxCollateral;
             MaxDuration = maxDuration;
-            BlockTTL = blockTTL;
-            BlockMI = blockMI;
-            BlockMN = blockMN;
         }
 
         public DateTime DeployDateTimeUtc { get; }
@@ -50,8 +45,5 @@ namespace DistTestCore.Codex
         public int MinPrice { get; }
         public int MaxCollateral { get; }
         public int MaxDuration { get; }
-        public int BlockTTL { get; }
-        public int BlockMI { get; }
-        public int BlockMN { get; }
     }
 }

DistTestCore/Codex/CodexStartupConfig.cs

@@ -1,5 +1,4 @@
 using DistTestCore.Marketplace;
-using DistTestCore.Metrics;
 using KubernetesWorkflow;
 
 namespace DistTestCore.Codex
@@ -15,11 +14,9 @@ namespace DistTestCore.Codex
         public Location Location { get; set; }
         public CodexLogLevel LogLevel { get; }
         public ByteSize? StorageQuota { get; set; }
-        public MetricsMode MetricsMode { get; set; }
+        public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
         public string? BootstrapSpr { get; set; }
         public int? BlockTTL { get; set; }
-        public TimeSpan? BlockMaintenanceInterval { get; set; }
-        public int? BlockMaintenanceNumber { get; set; }
     }
 }

DistTestCore/CodexNodeGroup.cs

@@ -14,13 +14,12 @@ namespace DistTestCore
     {
         private readonly TestLifecycle lifecycle;
 
-        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers[] containers, ICodexNodeFactory codexNodeFactory)
+        public CodexNodeGroup(TestLifecycle lifecycle, CodexSetup setup, RunningContainers containers, ICodexNodeFactory codexNodeFactory)
         {
             this.lifecycle = lifecycle;
             Setup = setup;
             Containers = containers;
-            Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
-            Version = new CodexDebugVersionResponse();
+            Nodes = containers.Containers.Select(c => CreateOnlineCodexNode(c, codexNodeFactory)).ToArray();
         }
 
         public IOnlineCodexNode this[int index]
@@ -45,9 +44,8 @@ namespace DistTestCore
         }
 
         public CodexSetup Setup { get; private set; }
-        public RunningContainers[] Containers { get; private set; }
+        public RunningContainers Containers { get; private set; }
         public OnlineCodexNode[] Nodes { get; private set; }
-        public CodexDebugVersionResponse Version { get; private set; }
 
         public IEnumerator<IOnlineCodexNode> GetEnumerator()
         {
@@ -66,17 +64,14 @@ namespace DistTestCore
 
         public void EnsureOnline()
         {
-            foreach (var node in Nodes) node.EnsureOnlineGetVersionResponse();
-            var versionResponses = Nodes.Select(n => n.Version);
-
-            var first = versionResponses.First();
-            if (!versionResponses.All(v => v.version == first.version && v.revision == first.revision))
+            foreach (var node in Nodes)
             {
-                throw new Exception("Inconsistent version information received from one or more Codex nodes: " +
-                    string.Join(",", versionResponses.Select(v => v.ToString())));
+                var debugInfo = node.CodexAccess.GetDebugInfo();
+                var nodePeerId = debugInfo.id;
+                var nodeName = node.CodexAccess.Container.Name;
+                lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
+                lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
             }
-
-            Version = first;
         }
 
         private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)

DistTestCore/CodexSetup.cs

@@ -10,9 +10,6 @@ namespace DistTestCore
         ICodexSetup At(Location location);
         ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
         ICodexSetup WithStorageQuota(ByteSize storageQuota);
-        ICodexSetup WithBlockTTL(TimeSpan duration);
-        ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration);
-        ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks);
         ICodexSetup EnableMetrics();
         ICodexSetup EnableMarketplace(TestToken initialBalance);
         ICodexSetup EnableMarketplace(TestToken initialBalance, Ether initialEther);
@@ -53,27 +50,9 @@ namespace DistTestCore
             return this;
         }
 
-        public ICodexSetup WithBlockTTL(TimeSpan duration)
-        {
-            BlockTTL = Convert.ToInt32(duration.TotalSeconds);
-            return this;
-        }
-
-        public ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration)
-        {
-            BlockMaintenanceInterval = duration;
-            return this;
-        }
-
-        public ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks)
-        {
-            BlockMaintenanceNumber = numberOfBlocks;
-            return this;
-        }
-
         public ICodexSetup EnableMetrics()
         {
-            MetricsMode = Metrics.MetricsMode.Record;
+            MetricsEnabled = true;
             return this;
         }
 

DistTestCore/CodexStarter.cs

@@ -8,8 +8,8 @@ namespace DistTestCore
 {
     public class CodexStarter : BaseStarter
     {
-        public CodexStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
+        public CodexStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+            : base(lifecycle, workflowCreator)
         {
         }
 
@@ -22,7 +22,6 @@ namespace DistTestCore
             var gethStartResult = lifecycle.GethStarter.BringOnlineMarketplaceFor(codexSetup);
 
             var startupConfig = CreateStartupConfig(gethStartResult, codexSetup);
-
             var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location);
 
             var metricAccessFactory = CollectMetrics(codexSetup, containers);
@@ -30,16 +29,9 @@ namespace DistTestCore
             var codexNodeFactory = new CodexNodeFactory(lifecycle, metricAccessFactory, gethStartResult.MarketplaceAccessFactory);
 
             var group = CreateCodexGroup(codexSetup, containers, codexNodeFactory);
-            lifecycle.SetCodexVersion(group.Version);
-
-            var nl = Environment.NewLine;
-            var podInfos = string.Join(nl, containers.Containers().Select(c => $"Container: '{c.Name}' runs at '{c.Pod.PodInfo.K8SNodeName}'={c.Pod.PodInfo.Ip}"));
-            LogEnd($"Started {codexSetup.NumberOfNodes} nodes " +
-                $"of image '{containers.Containers().First().Recipe.Image}' " +
-                $"and version '{group.Version}'{nl}" +
-                podInfos);
+            var podInfo = group.Containers.RunningPod.PodInfo;
+            LogEnd($"Started {codexSetup.NumberOfNodes} nodes of image '{containers.Containers.First().Recipe.Image}' at location '{podInfo.K8SNodeName}'={podInfo.Ip}. They are: {group.Describe()}");
             LogSeparator();
-
             return group;
         }
 
@@ -47,11 +39,7 @@ namespace DistTestCore
         {
             LogStart($"Stopping {group.Describe()}...");
             var workflow = CreateWorkflow();
-            foreach (var c in group.Containers)
-            {
-                StopCrashWatcher(c);
-                workflow.Stop(c);
-            }
+            workflow.Stop(group.Containers);
             RunningGroups.Remove(group);
             LogEnd("Stopped.");
         }
@@ -64,23 +52,17 @@ namespace DistTestCore
             RunningGroups.Clear();
         }
 
-        public void DownloadLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
+        public void DownloadLog(RunningContainer container, ILogHandler logHandler)
         {
             var workflow = CreateWorkflow();
-            workflow.DownloadContainerLog(container, logHandler, tailLines);
+            workflow.DownloadContainerLog(container, logHandler);
         }
 
-        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers[] containers)
+        private IMetricsAccessFactory CollectMetrics(CodexSetup codexSetup, RunningContainers containers)
         {
-            if (codexSetup.MetricsMode == MetricsMode.None) return new MetricsUnavailableAccessFactory();
+            if (!codexSetup.MetricsEnabled) return new MetricsUnavailableAccessFactory();
 
             var runningContainers = lifecycle.PrometheusStarter.CollectMetricsFor(containers);
-
-            if (codexSetup.MetricsMode == MetricsMode.Dashboard)
-            {
-                lifecycle.GrafanaStarter.StartDashboard(runningContainers.Containers.First(), codexSetup);
-            }
-
             return new CodexNodeMetricsAccessFactory(lifecycle, runningContainers);
         }
 
@@ -93,66 +75,28 @@ namespace DistTestCore
             return startupConfig;
         }
 
-        private RunningContainers[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
+        private RunningContainers StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, Location location)
         {
-            var result = new List<RunningContainers>();
-            var recipe = new CodexContainerRecipe();
-            for (var i = 0; i < numberOfNodes; i++)
-            {
-                var workflow = CreateWorkflow();
-                var rc = workflow.Start(1, location, recipe, startupConfig);
-                CreateCrashWatcher(workflow, rc);
-                result.Add(rc);
-            }
-            return result.ToArray();
+            var workflow = CreateWorkflow();
+            return workflow.Start(numberOfNodes, location, new CodexContainerRecipe(), startupConfig);
         }
 
-        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers[] runningContainers, CodexNodeFactory codexNodeFactory)
+        private CodexNodeGroup CreateCodexGroup(CodexSetup codexSetup, RunningContainers runningContainers, CodexNodeFactory codexNodeFactory)
         {
             var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
             RunningGroups.Add(group);
-
-            try
-            {
-                Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
-            }
-            catch
-            {
-                CodexNodesNotOnline(runningContainers);
-                throw;
-            }
-
+            Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
             return group;
         }
 
-        private void CodexNodesNotOnline(RunningContainers[] runningContainers)
-        {
-            Log("Codex nodes failed to start");
-            foreach (var container in runningContainers.Containers()) lifecycle.DownloadLog(container);
-        }
-
         private StartupWorkflow CreateWorkflow()
         {
-            return lifecycle.WorkflowCreator.CreateWorkflow();
+            return workflowCreator.CreateWorkflow();
         }
 
         private void LogSeparator()
         {
             Log("----------------------------------------------------------------------------");
         }
-
-        private void CreateCrashWatcher(StartupWorkflow workflow, RunningContainers rc)
-        {
-            var c = rc.Containers.Single();
-            c.CrashWatcher = workflow.CreateCrashWatcher(c);
-        }
-
-        private void StopCrashWatcher(RunningContainers containers)
-        {
-            foreach (var c in containers.Containers)
-            {
-                c.CrashWatcher?.Stop();
-            }
-        }
     }
 }

DistTestCore/Configuration.cs

@@ -1,6 +1,5 @@
 using DistTestCore.Codex;
 using KubernetesWorkflow;
-using System.Net.NetworkInformation;
 using Utils;
 
 namespace DistTestCore
@@ -12,8 +11,7 @@ namespace DistTestCore
         private readonly bool logDebug;
         private readonly string dataFilesPath;
         private readonly CodexLogLevel codexLogLevel;
-        private readonly string k8sNamespacePrefix;
-        private static RunnerLocation? runnerLocation = null;
+        private readonly TestRunnerLocation runnerLocation;
 
         public Configuration()
         {
@@ -22,23 +20,23 @@ namespace DistTestCore
             logDebug = GetEnvVarOrDefault("LOGDEBUG", "false").ToLowerInvariant() == "true";
             dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles");
             codexLogLevel = ParseEnum.Parse<CodexLogLevel>(GetEnvVarOrDefault("LOGLEVEL", nameof(CodexLogLevel.Trace)));
-            k8sNamespacePrefix = "ct-";
+            runnerLocation = ParseEnum.Parse<TestRunnerLocation>(GetEnvVarOrDefault("RUNNERLOCATION", nameof(TestRunnerLocation.ExternalToCluster)));
         }
 
-        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, string k8sNamespacePrefix)
+        public Configuration(string? kubeConfigFile, string logPath, bool logDebug, string dataFilesPath, CodexLogLevel codexLogLevel, TestRunnerLocation runnerLocation)
         {
             this.kubeConfigFile = kubeConfigFile;
             this.logPath = logPath;
             this.logDebug = logDebug;
             this.dataFilesPath = dataFilesPath;
             this.codexLogLevel = codexLogLevel;
-            this.k8sNamespacePrefix = k8sNamespacePrefix;
+            this.runnerLocation = runnerLocation;
         }
 
         public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet)
         {
             return new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: k8sNamespacePrefix,
+                k8sNamespacePrefix: "ct-",
                 kubeConfigFile: kubeConfigFile,
                 operationTimeout: timeSet.K8sOperationTimeout(),
                 retryDelay: timeSet.WaitForK8sServiceDelay()
@@ -60,14 +58,14 @@ namespace DistTestCore
             return codexLogLevel;
         }
 
+        public TestRunnerLocation GetTestRunnerLocation()
+        {
+            return runnerLocation;
+        }
+
         public Address GetAddress(RunningContainer container)
         {
-            if (runnerLocation == null)
-            {
-                runnerLocation = RunnerLocationUtils.DetermineRunnerLocation(container);
-            }
-            
-            if (runnerLocation == RunnerLocation.InternalToCluster)
+            if (GetTestRunnerLocation() == TestRunnerLocation.InternalToCluster)
             {
                 return container.ClusterInternalAddress;
             }
@@ -89,56 +87,9 @@ namespace DistTestCore
         }
     }
 
-    public enum RunnerLocation
+    public enum TestRunnerLocation
     {
         ExternalToCluster,
         InternalToCluster,
     }
-
-    public static class RunnerLocationUtils
-    {
-        private static bool alreadyDidThat = false;
-
-        public static RunnerLocation DetermineRunnerLocation(RunningContainer container)
-        {
-            // We want to be sure we don't ping more often than strictly necessary.
-            // If we have already determined the location during this application
-            // lifetime, don't do it again.
-            if (alreadyDidThat) throw new Exception("We already did that.");
-            alreadyDidThat = true;
-
-            if (PingHost(container.Pod.PodInfo.Ip))
-            {
-                return RunnerLocation.InternalToCluster;
-            }
-            if (PingHost(Format(container.ClusterExternalAddress)))
-            {
-                return RunnerLocation.ExternalToCluster;
-            }
-
-            throw new Exception("Unable to determine runner location.");
-        }
-
-        private static string Format(Address host)
-        {
-            return host.Host
-                .Replace("http://", "")
-                .Replace("https://", "");
-        }
-
-        private static bool PingHost(string host)
-        {
-            try
-            {
-                using var pinger = new Ping();
-                PingReply reply = pinger.Send(host);
-                return reply.Status == IPStatus.Success;
-            }
-            catch (PingException)
-            {
-            }
-
-            return false;
-        }
-    }
 }

DistTestCore/DefaultContainerRecipe.cs

@@ -1,40 +0,0 @@
-using KubernetesWorkflow;
-using Logging;
-
-namespace DistTestCore
-{
-    public abstract class DefaultContainerRecipe : ContainerRecipeFactory
-    {
-        public static string TestsType { get; set; } = "NotSet";
-        public static ApplicationIds? ApplicationIds { get; set; } = null;
-
-        protected abstract void InitializeRecipe(StartupConfig config);
-
-        protected override void Initialize(StartupConfig config)
-        {
-            Add("tests-type", TestsType);
-            Add("runid", NameUtils.GetRunId());
-            Add("testid", NameUtils.GetTestId());
-            Add("category", NameUtils.GetCategoryName());
-            Add("fixturename", NameUtils.GetRawFixtureName());
-            Add("testname", NameUtils.GetTestMethodName());
-
-            if (ApplicationIds != null)
-            {
-                Add("codexid", ApplicationIds.CodexId);
-                Add("gethid", ApplicationIds.GethId);
-                Add("prometheusid", ApplicationIds.PrometheusId);
-                Add("codexcontractsid", ApplicationIds.CodexContractsId);
-                Add("grafanaid", ApplicationIds.GrafanaId);
-            }
-            Add("app", AppName);
-
-            InitializeRecipe(config);
-        }
-
-        private void Add(string name, string value)
-        {
-            AddPodLabel(name, value);
-        }
-    }
-}

DistTestCore/DistTest.cs

@@ -13,11 +13,9 @@ namespace DistTestCore
     [Parallelizable(ParallelScope.All)]
     public abstract class DistTest
     {
-        private const string TestsType = "dist-tests";
         private readonly Configuration configuration = new Configuration();
         private readonly Assembly[] testAssemblies;
         private readonly FixtureLog fixtureLog;
-        private readonly StatusLog statusLog;
         private readonly object lifecycleLock = new object();
         private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
 
@@ -26,28 +24,25 @@ namespace DistTestCore
             var assemblies = AppDomain.CurrentDomain.GetAssemblies();
             testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
 
-            var logConfig = configuration.GetLogConfig();
-            var startTime = DateTime.UtcNow;
-            fixtureLog = new FixtureLog(logConfig, startTime);
-            statusLog = new StatusLog(logConfig, startTime);
+            fixtureLog = new FixtureLog(configuration.GetLogConfig());
+
+            PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
+            PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
         }
 
+        public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; }
+        public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; }
+
         [OneTimeSetUp]
         public void GlobalSetup()
         {
-            fixtureLog.Log($"Codex Distributed Tests are starting...");
-            fixtureLog.Log($"Codex image: '{new CodexContainerRecipe().Image}'");
-            fixtureLog.Log($"CodexContracts image: '{new CodexContractsContainerRecipe().Image}'");
-            fixtureLog.Log($"Prometheus image: '{new PrometheusContainerRecipe().Image}'");
-            fixtureLog.Log($"Geth image: '{new GethContainerRecipe().Image}'");
-
             // Previous test run may have been interrupted.
             // Begin by cleaning everything up.
             try
             {
                 Stopwatch.Measure(fixtureLog, "Global setup", () =>
                 {
-                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), string.Empty);
+                    var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
                     wc.CreateWorkflow().DeleteAllResources();
                 });                
             }
@@ -59,6 +54,9 @@ namespace DistTestCore
             }
 
             fixtureLog.Log("Global setup cleanup successful");
+            fixtureLog.Log($"Codex image: '{CodexContainerRecipe.DockerImage}'");
+            fixtureLog.Log($"Prometheus image: '{PrometheusContainerRecipe.DockerImage}'");
+            fixtureLog.Log($"Geth image: '{GethContainerRecipe.DockerImage}'");
         }
 
         [SetUp]
@@ -169,26 +167,6 @@ namespace DistTestCore
             GetTestLog().Debug(msg);
         }
 
-        public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers()
-        {
-            return new PeerConnectionTestHelpers(GetTestLog());
-        }
-
-        public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers()
-        {
-            return new PeerDownloadTestHelpers(GetTestLog(), Get().FileManager);
-        }
-
-        public PeerFetchTestHelpers CreatePeerFetchTestHelpers()
-        {
-            return new PeerFetchTestHelpers(GetTestLog(), Get().FileManager);
-        }
-
-        public void Measure(string name, Action action)
-        {
-            Stopwatch.Measure(Get().Log, name, action);
-        }
-
         protected CodexSetup CreateCodexSetup(int numberOfNodes)
         {
             return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
@@ -205,16 +183,11 @@ namespace DistTestCore
         private void CreateNewTestLifecycle()
         {
             var testName = GetCurrentTestName();
-            fixtureLog.WriteLogTag();
             Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
             {
                 lock (lifecycleLock)
                 {
-                    var testNamespace = Guid.NewGuid().ToString();
-                    var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace);
-                    lifecycles.Add(testName, lifecycle);
-                    DefaultContainerRecipe.TestsType = TestsType;
-                    DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
+                    lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
                 }
             });
         }
@@ -222,10 +195,7 @@ namespace DistTestCore
         private void DisposeTestLifecycle()
         {
             var lifecycle = Get();
-            var testResult = GetTestResult();
-            var testDuration = lifecycle.GetTestDuration();
-            fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
-            statusLog.ConcludeTest(testResult, testDuration, lifecycle.GetApplicationIds());
+            fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
             Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
             {
                 lifecycle.Log.EndTest();

DistTestCore/DistTestCore.csproj

@@ -10,14 +10,6 @@
   <PropertyGroup Condition="'$(IsArm64)'=='true'">
     <DefineConstants>Arm64</DefineConstants>
   </PropertyGroup>
-  <ItemGroup>
-    <None Remove="Metrics\dashboard.json" />
-  </ItemGroup>
-  <ItemGroup>
-    <EmbeddedResource Include="Metrics\dashboard.json">
-      <CopyToOutputDirectory>Never</CopyToOutputDirectory>
-    </EmbeddedResource>
-  </ItemGroup>
 
   <ItemGroup>
 	  <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />

DistTestCore/GethStarter.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Marketplace;
+using KubernetesWorkflow;
 
 namespace DistTestCore
 {
@@ -7,13 +8,13 @@ namespace DistTestCore
         private readonly MarketplaceNetworkCache marketplaceNetworkCache;
         private readonly GethCompanionNodeStarter companionNodeStarter;
 
-        public GethStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
+        public GethStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+            : base(lifecycle, workflowCreator)
         {
             marketplaceNetworkCache = new MarketplaceNetworkCache(
-                new GethBootstrapNodeStarter(lifecycle),
-                new CodexContractsStarter(lifecycle));
-            companionNodeStarter = new GethCompanionNodeStarter(lifecycle);
+                new GethBootstrapNodeStarter(lifecycle, workflowCreator),
+                new CodexContractsStarter(lifecycle, workflowCreator));
+            companionNodeStarter = new GethCompanionNodeStarter(lifecycle, workflowCreator);
         }
 
         public GethStartResult BringOnlineMarketplaceFor(CodexSetup codexSetup)

DistTestCore/GrafanaStarter.cs

@@ -1,192 +0,0 @@
-using DistTestCore.Metrics;
-using IdentityModel.Client;
-using KubernetesWorkflow;
-using Newtonsoft.Json;
-using System.Reflection;
-
-namespace DistTestCore
-{
-    public class GrafanaStarter : BaseStarter
-    {
-        private const string StorageQuotaThresholdReplaceToken = "\"<CODEX_STORAGEQUOTA>\"";
-        private const string BytesUsedGraphAxisSoftMaxReplaceToken = "\"<CODEX_BYTESUSED_SOFTMAX>\"";
-
-        public GrafanaStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
-        {
-        }
-
-        public GrafanaStartInfo StartDashboard(RunningContainer prometheusContainer, CodexSetup codexSetup)
-        {
-            LogStart($"Starting dashboard server");
-
-            var grafanaContainer = StartGrafanaContainer();
-            var grafanaAddress = lifecycle.Configuration.GetAddress(grafanaContainer);
-
-            var http = new Http(lifecycle.Log, new DefaultTimeSet(), grafanaAddress, "api/", AddBasicAuth);
-
-            Log("Connecting datasource...");
-            AddDataSource(http, prometheusContainer);
-
-            Log("Uploading dashboard configurations...");
-            var jsons = ReadEachDashboardJsonFile(codexSetup);
-            var dashboardUrls = jsons.Select(j => UploadDashboard(http, grafanaContainer, j)).ToArray();
-
-            LogEnd("Dashboard server started.");
-
-            return new GrafanaStartInfo(dashboardUrls, grafanaContainer);
-        }
-
-        private RunningContainer StartGrafanaContainer()
-        {
-            var startupConfig = new StartupConfig();
-
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
-            var grafanaContainers = workflow.Start(1, Location.Unspecified, new GrafanaContainerRecipe(), startupConfig);
-            if (grafanaContainers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 dashboard container to be created.");
-
-            return grafanaContainers.Containers.First();
-        }
-
-        private void AddBasicAuth(HttpClient client)
-        {
-            client.SetBasicAuthentication(
-                GrafanaContainerRecipe.DefaultAdminUser,
-                GrafanaContainerRecipe.DefaultAdminPassword);
-        }
-
-        private static void AddDataSource(Http http, RunningContainer prometheusContainer)
-        {
-            var prometheusAddress = prometheusContainer.ClusterExternalAddress;
-            var prometheusUrl = prometheusAddress.Host + ":" + prometheusAddress.Port;
-            var response = http.HttpPostJson<GrafanaDataSourceRequest, GrafanaDataSourceResponse>("datasources", new GrafanaDataSourceRequest
-            {
-                uid = "c89eaad3-9184-429f-ac94-8ba0b1824dbb",
-                name = "CodexPrometheus",
-                type = "prometheus",
-                url = prometheusUrl,
-                access = "proxy",
-                basicAuth = false,
-                jsonData = new GrafanaDataSourceJsonData
-                {
-                    httpMethod = "POST"
-                }
-            });
-
-            if (response.message != "Datasource added")
-            {
-                throw new Exception("Test infra failure: Failed to add datasource to dashboard: " + response.message);
-            }
-        }
-
-        public static string UploadDashboard(Http http, RunningContainer grafanaContainer, string dashboardJson)
-        {
-            var request = GetDashboardCreateRequest(dashboardJson);
-            var response = http.HttpPostString("dashboards/db", request);
-            var jsonResponse = JsonConvert.DeserializeObject<GrafanaPostDashboardResponse>(response);
-            if (jsonResponse == null || string.IsNullOrEmpty(jsonResponse.url)) throw new Exception("Failed to upload dashboard.");
-
-            var grafanaAddress = grafanaContainer.ClusterExternalAddress;
-            return grafanaAddress.Host + ":" + grafanaAddress.Port + jsonResponse.url;
-        }
-
-        private static string[] ReadEachDashboardJsonFile(CodexSetup codexSetup)
-        {
-            var assembly = Assembly.GetExecutingAssembly();
-            var resourceNames = new[]
-            {
-                "DistTestCore.Metrics.dashboard.json"
-            };
-
-            return resourceNames.Select(r => GetManifestResource(assembly, r, codexSetup)).ToArray();
-        }
-
-        private static string GetManifestResource(Assembly assembly, string resourceName, CodexSetup codexSetup)
-        {
-            using var stream = assembly.GetManifestResourceStream(resourceName);
-            if (stream == null) throw new Exception("Unable to find resource " + resourceName);
-            using var reader = new StreamReader(stream);
-            return ApplyReplacements(reader.ReadToEnd(), codexSetup);
-        }
-
-        private static string ApplyReplacements(string input, CodexSetup codexSetup)
-        {
-            var quotaString = GetQuotaString(codexSetup);
-            var softMaxString = GetSoftMaxString(codexSetup);
-
-            return input
-                .Replace(StorageQuotaThresholdReplaceToken, quotaString)
-                .Replace(BytesUsedGraphAxisSoftMaxReplaceToken, softMaxString);
-        }
-
-        private static string GetQuotaString(CodexSetup codexSetup)
-        {
-            return GetCodexStorageQuotaInBytes(codexSetup).ToString();
-        }
-
-        private static string GetSoftMaxString(CodexSetup codexSetup)
-        {
-            var quota = GetCodexStorageQuotaInBytes(codexSetup);
-            var softMax = Convert.ToInt64(quota * 1.1); // + 10%, for nice viewing.
-            return softMax.ToString();
-        }
-
-        private static long GetCodexStorageQuotaInBytes(CodexSetup codexSetup)
-        {
-            if (codexSetup.StorageQuota != null) return codexSetup.StorageQuota.SizeInBytes;
-
-            // Codex default: 8GB
-            return 8.GB().SizeInBytes;
-        }
-
-        private static string GetDashboardCreateRequest(string dashboardJson)
-        {
-            return $"{{\"dashboard\": {dashboardJson} ,\"message\": \"Default Codex Dashboard\",\"overwrite\": false}}";
-        }
-    }
-
-    public class GrafanaStartInfo
-    {
-        public GrafanaStartInfo(string[] dashboardUrls, RunningContainer container)
-        {
-            DashboardUrls = dashboardUrls;
-            Container = container;
-        }
-
-        public string[] DashboardUrls { get; }
-        public RunningContainer Container { get; }
-    }
-
-    public class GrafanaDataSourceRequest
-    {
-        public string uid { get; set; } = string.Empty;
-        public string name { get; set; } = string.Empty;
-        public string type { get; set; } = string.Empty;
-        public string url { get; set; } = string.Empty;
-        public string access { get; set; } = string.Empty;
-        public bool basicAuth { get; set; }
-        public GrafanaDataSourceJsonData jsonData { get; set; } = new();
-    }
-
-    public class GrafanaDataSourceResponse
-    {
-        public int id { get; set; }
-        public string message { get; set; } = string.Empty;
-        public string name { get; set; } = string.Empty;
-    }
-
-    public class GrafanaDataSourceJsonData
-    {
-        public string httpMethod { get; set; } = string.Empty;
-    }
-
-    public class GrafanaPostDashboardResponse
-    {
-        public int id { get; set; }
-        public string slug { get; set; } = string.Empty;
-        public string status { get; set; } = string.Empty;
-        public string uid { get; set; } = string.Empty;
-        public string url { get; set; } = string.Empty;
-        public int version { get; set; }
-    }
-}

DistTestCore/Helpers/FullConnectivityHelper.cs

@@ -1,217 +0,0 @@
-using DistTestCore.Codex;
-using Logging;
-using NUnit.Framework;
-using Utils;
-
-namespace DistTestCore.Helpers
-{
-    public interface IFullConnectivityImplementation
-    {
-        string Description();
-        string ValidateEntry(FullConnectivityHelper.Entry entry, FullConnectivityHelper.Entry[] allEntries);
-        FullConnectivityHelper.PeerConnectionState Check(FullConnectivityHelper.Entry from, FullConnectivityHelper.Entry to);
-    }
-
-    public class FullConnectivityHelper
-    {
-        private static string Nl = Environment.NewLine;
-        private readonly BaseLog log;
-        private readonly IFullConnectivityImplementation implementation;
-
-        public FullConnectivityHelper(BaseLog log, IFullConnectivityImplementation implementation)
-        {
-            this.log = log;
-            this.implementation = implementation;
-        }
-
-        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
-        {
-            AssertFullyConnected(nodes.ToArray());
-        }
-
-        private void AssertFullyConnected(CodexAccess[] nodes)
-        {
-            Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
-            var entries = CreateEntries(nodes);
-            var pairs = CreatePairs(entries);
-
-            RetryWhilePairs(pairs, () =>
-            {
-                CheckAndRemoveSuccessful(pairs);
-            });
-
-            if (pairs.Any())
-            {
-                var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
-
-                Log($"Connections failed:{Nl}{pairDetails}");
-
-                Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
-            }
-            else
-            {
-                Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
-            }
-        }
-
-        private static void RetryWhilePairs(List<Pair> pairs, Action action)
-        {
-            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2);
-            while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
-            {
-                action();
-
-                Time.Sleep(TimeSpan.FromSeconds(2));
-            }
-        }
-
-        private void CheckAndRemoveSuccessful(List<Pair> pairs)
-        {
-            // For large sets, don't try and do all of them at once.
-            var selectedPair = pairs.Take(20).ToArray();
-            var pairDetails = new List<string>();
-
-            foreach (var pair in selectedPair)
-            {
-                pair.Check();
-
-                if (pair.Success)
-                {
-                    pairDetails.AddRange(pair.GetResultMessages());
-                    pairs.Remove(pair);
-                }
-            }
-
-            Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
-        }
-
-        private Entry[] CreateEntries(CodexAccess[] nodes)
-        {
-            var entries = nodes.Select(n => new Entry(n)).ToArray();
-
-            var errors = entries
-                            .Select(e => implementation.ValidateEntry(e, entries))
-                            .Where(s => !string.IsNullOrEmpty(s))
-                            .ToArray();
-
-            if (errors.Any())
-            {
-                Assert.Fail("Some node entries failed to validate: " + string.Join(Nl, errors));
-            }
-
-            return entries;
-        }
-
-        private List<Pair> CreatePairs(Entry[] entries)
-        {
-            return CreatePairsIterator(entries).ToList();
-        }
-
-        private IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
-        {
-            for (var x = 0; x < entries.Length; x++)
-            {
-                for (var y = x + 1; y < entries.Length; y++)
-                {
-                    yield return new Pair(implementation, entries[x], entries[y]);
-                }
-            }
-        }
-
-        private void Log(string msg)
-        {
-            log.Log(msg);
-        }
-
-        public class Entry
-        {
-            public Entry(CodexAccess node)
-            {
-                Node = node;
-                Response = node.GetDebugInfo();
-            }
-
-            public CodexAccess Node { get; }
-            public CodexDebugResponse Response { get; }
-
-            public override string ToString()
-            {
-                if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
-                return Response.id;
-            }
-        }
-
-        public enum PeerConnectionState
-        {
-            Unknown,
-            Connection,
-            NoConnection,
-        }
-
-        public class Pair
-        {
-            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
-            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
-            private readonly IFullConnectivityImplementation implementation;
-
-            public Pair(IFullConnectivityImplementation implementation, Entry a, Entry b)
-            {
-                this.implementation = implementation;
-                A = a;
-                B = b;
-            }
-
-            public Entry A { get; }
-            public Entry B { get; }
-            public PeerConnectionState AKnowsB { get; private set; }
-            public PeerConnectionState BKnowsA { get; private set; }
-            public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
-            public bool Inconclusive { get { return AKnowsB == PeerConnectionState.Unknown || BKnowsA == PeerConnectionState.Unknown; } }
-
-            public void Check()
-            {
-                aToBTime = Measure(() => AKnowsB = Check(A, B));
-                bToATime = Measure(() => BKnowsA = Check(B, A));
-            }
-
-            public override string ToString()
-            {
-                return $"[{string.Join(",", GetResultMessages())}]";
-            }
-
-            public string[] GetResultMessages()
-            {
-                var aName = A.ToString();
-                var bName = B.ToString();
-
-                return new[]
-                {
-                    $"[{aName} --> {bName}] = {AKnowsB} ({aToBTime.TotalSeconds} seconds)",
-                    $"[{aName} <-- {bName}] = {BKnowsA} ({bToATime.TotalSeconds} seconds)"
-                };
-            }
-
-            private static TimeSpan Measure(Action action)
-            {
-                var start = DateTime.UtcNow;
-                action();
-                return DateTime.UtcNow - start;
-            }
-
-            private PeerConnectionState Check(Entry from, Entry to)
-            {
-                Thread.Sleep(10);
-
-                try
-                {
-                    return implementation.Check(from, to);
-                }
-                catch
-                {
-                    // Didn't get a conclusive answer. Try again later.
-                    return PeerConnectionState.Unknown;
-                }
-            }
-        }
-    }
-}

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -1,71 +1,257 @@
 using DistTestCore.Codex;
-using Logging;
-using static DistTestCore.Helpers.FullConnectivityHelper;
+using NUnit.Framework;
+using Utils;
 
 namespace DistTestCore.Helpers
 {
-    public class PeerConnectionTestHelpers : IFullConnectivityImplementation
+    public class PeerConnectionTestHelpers
     {
-        private readonly FullConnectivityHelper helper;
+        private readonly Random random = new Random();
+        private readonly DistTest test;
 
-        public PeerConnectionTestHelpers(BaseLog log)
+        public PeerConnectionTestHelpers(DistTest test)
         {
-            helper = new FullConnectivityHelper(log, this);
+            this.test = test;
         }
 
         public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
         {
-            AssertFullyConnected(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess));
-        }
+            var n = nodes.ToArray();
 
-        public void AssertFullyConnected(IEnumerable<CodexAccess> nodes)
-        {
-            helper.AssertFullyConnected(nodes);
-        }
+            AssertFullyConnected(n);
 
-        public string Description()
-        {
-            return "Peer Discovery";
-        }
-
-        public string ValidateEntry(Entry entry, Entry[] allEntries)
-        {
-            var result = string.Empty;
-            foreach (var peer in entry.Response.table.nodes)
+            for (int i = 0; i < 5; i++)
             {
-                var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
-                if (expected != peer.address)
+                Time.Sleep(TimeSpan.FromSeconds(30));
+                AssertFullyConnected(n);
+            }
+        }
+
+        private void AssertFullyConnected(IOnlineCodexNode[] nodes)
+        {
+            test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            var entries = CreateEntries(nodes);
+            var pairs = CreatePairs(entries);
+
+            RetryWhilePairs(pairs, () =>
+            {
+                CheckAndRemoveSuccessful(pairs);
+            });
+
+            if (pairs.Any())
+            {
+                test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
+                Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
+                test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
+            }
+            else
+            {
+                test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
+            }
+        }
+
+        private static void RetryWhilePairs(List<Pair> pairs, Action action)
+        {
+            var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(30);
+            while (pairs.Any() && timeout > DateTime.UtcNow)
+            {
+                action();
+
+                if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(2));
+            }
+        }
+
+        private void CheckAndRemoveSuccessful(List<Pair> pairs)
+        {
+            var checkTasks = pairs.Select(p => Task.Run(() =>
+            {
+                ApplyRandomDelay();
+                p.Check();
+            })).ToArray();
+
+            Task.WaitAll(checkTasks);
+
+            foreach (var pair in pairs.ToArray())
+            {
+                if (pair.Success)
                 {
-                    result += $"Node:{entry.Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'. ";
+                    test.Log(pair.GetMessage());
+                    pairs.Remove(pair);
                 }
             }
-            return result;
         }
 
-        public PeerConnectionState Check(Entry from, Entry to)
+        private static Entry[] CreateEntries(IOnlineCodexNode[] nodes)
         {
-            var peerId = to.Response.id;
+            var entries = nodes.Select(n => new Entry(n)).ToArray();
+            var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
 
-            var response = from.Node.GetDebugPeer(peerId);
-            if (!response.IsPeerFound)
+            if (incorrectDiscoveryEndpoints.Any())
             {
-                return PeerConnectionState.NoConnection;
+                Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
+                    string.Join(Environment.NewLine, incorrectDiscoveryEndpoints));
             }
-            if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
-            {
-                return PeerConnectionState.Connection;
-            }
-            return PeerConnectionState.Unknown;
+
+            return entries;
         }
 
-        private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
+        private static List<Pair> CreatePairs(Entry[] entries)
         {
-            var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
-            if (peer == null) return $"peerId: {node.peerId} is not known.";
+            return CreatePairsIterator(entries).ToList();
+        }
 
-            var ip = peer.Node.Container.Pod.PodInfo.Ip;
-            var discPort = peer.Node.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
-            return $"{ip}:{discPort.Number}";
+        private static IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
+        {
+            for (var x = 0; x < entries.Length; x++)
+            {
+                for (var y = x + 1; y < entries.Length; y++)
+                {
+                    yield return new Pair(entries[x], entries[y]);
+                }
+            }
+        }
+
+        private void ApplyRandomDelay()
+        {
+            // Calling all the nodes all at the same time is not exactly nice.
+            Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000)));
+        }
+
+        public class Entry
+        {
+            public Entry(IOnlineCodexNode node)
+            {
+                Node = node;
+                Response = node.GetDebugInfo();
+            }
+
+            public IOnlineCodexNode Node { get; }
+            public CodexDebugResponse Response { get; }
+
+            public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
+            {
+                foreach (var peer in Response.table.nodes)
+                {
+                    var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
+                    if (expected != peer.address)
+                    {
+                        yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'";
+                    }
+                }
+            }
+
+            public override string ToString()
+            {
+                if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
+                return Response.id;
+            }
+
+            private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
+            {
+                var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
+                if (peer == null) return $"peerId: {node.peerId} is not known.";
+
+                var n = (OnlineCodexNode)peer.Node;
+                var ip = n.CodexAccess.Container.Pod.PodInfo.Ip;
+                var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
+                return $"{ip}:{discPort.Number}";
+            }
+        }
+
+        public enum PeerConnectionState
+        {
+            Unknown,
+            Connection,
+            NoConnection,
+        }
+
+        public class Pair
+        {
+            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
+            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
+
+            public Pair(Entry a, Entry b)
+            {
+                A = a;
+                B = b;
+            }
+
+            public Entry A { get; }
+            public Entry B { get; }
+            public PeerConnectionState AKnowsB { get; private set; }
+            public PeerConnectionState BKnowsA { get; private set; }
+            public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
+
+            public void Check()
+            {
+                aToBTime = Measure(() => AKnowsB = Knows(A, B));
+                bToATime = Measure(() => BKnowsA = Knows(B, A));
+            }
+
+            public string GetMessage()
+            {
+                return GetResultMessage() + GetTimePostfix();
+            }
+
+            public override string ToString()
+            {
+                return $"[{GetMessage()}]";
+            }
+
+            private string GetResultMessage()
+            {
+                var aName = A.ToString();
+                var bName = B.ToString();
+
+                if (Success)
+                {
+                    return $"{aName} and {bName} know each other.";
+                }
+
+                return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}";
+            }
+
+            private string GetTimePostfix()
+            {
+                var aName = A.ToString();
+                var bName = B.ToString();
+
+                return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
+            }
+
+            private static TimeSpan Measure(Action action)
+            {
+                var start = DateTime.UtcNow;
+                action();
+                return DateTime.UtcNow - start;
+            }
+
+            private PeerConnectionState Knows(Entry a, Entry b)
+            {
+                lock (a)
+                {
+                    var peerId = b.Response.id;
+
+                    try
+                    {
+                        var response = a.Node.GetDebugPeer(peerId);
+                        if (!response.IsPeerFound)
+                        {
+                            return PeerConnectionState.NoConnection;
+                        }
+                        if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
+                        {
+                            return PeerConnectionState.Connection;
+                        }
+                    }
+                    catch
+                    {
+                    }
+
+                    // Didn't get a conclusive answer. Try again later.
+                    return PeerConnectionState.Unknown;
+                }
+            }
         }
     }
 }

DistTestCore/Helpers/PeerDownloadTestHelpers.cs

@@ -1,88 +1,75 @@
 using DistTestCore.Codex;
-using Logging;
-using static DistTestCore.Helpers.FullConnectivityHelper;
+using NUnit.Framework;
 
 namespace DistTestCore.Helpers
 {
-    public class PeerDownloadTestHelpers : IFullConnectivityImplementation
+    public class PeerDownloadTestHelpers
     {
-        private readonly FullConnectivityHelper helper;
-        private readonly BaseLog log;
-        private readonly FileManager fileManager;
-        private ByteSize testFileSize;
+        private readonly DistTest test;
 
-        public PeerDownloadTestHelpers(BaseLog log, FileManager fileManager)
+        public PeerDownloadTestHelpers(DistTest test)
         {
-            helper = new FullConnectivityHelper(log, this);
-            testFileSize = 1.MB();
-            this.log = log;
-            this.fileManager = fileManager;
+            this.test = test;
         }
 
         public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
         {
-            AssertFullDownloadInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess), testFileSize);
-        }
+            test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            var start = DateTime.UtcNow;
 
-        public void AssertFullDownloadInterconnectivity(IEnumerable<CodexAccess> nodes, ByteSize testFileSize)
-        {
-            this.testFileSize = testFileSize;
-            helper.AssertFullyConnected(nodes);
-        }
-
-        public string Description()
-        {
-            return "Download Connectivity";
-        }
-
-        public string ValidateEntry(Entry entry, Entry[] allEntries)
-        {
-            return string.Empty;
-        }
-
-        public PeerConnectionState Check(Entry from, Entry to)
-        {
-            fileManager.PushFileSet();
-            var expectedFile = GenerateTestFile(from.Node, to.Node);
-
-            using var uploadStream = File.OpenRead(expectedFile.Filename);
-            var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(uploadStream));
-
-            try
+            foreach (var node in nodes)
             {
-                var downloadedFile = Stopwatch.Measure(log, "Download", () => DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded"));
+                var uploader = node;
+                var downloaders = nodes.Where(n => n != uploader).ToArray();
+
+                test.ScopedTestFiles(() =>
+                {
+                    PerformTest(uploader, downloaders, testFileSize);
+                });
+            }
+            
+            test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
+            var timeTaken = DateTime.UtcNow - start;
+
+            AssertTimePerMB(timeTaken, nodes.Count(), testFileSize);
+        }
+
+        private void AssertTimePerMB(TimeSpan timeTaken, int numberOfNodes, ByteSize size)
+        {
+            var numberOfDownloads = numberOfNodes * (numberOfNodes - 1);
+            var timePerDownload = timeTaken / numberOfDownloads;
+            float sizeInMB = size.ToMB();
+            var timePerMB = timePerDownload / sizeInMB;
+
+            test.Log($"Performed {numberOfDownloads} downloads of {size} in {timeTaken.TotalSeconds} seconds, for an average of {timePerMB.TotalSeconds} seconds per MB.");
+
+            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
+        }
+
+        private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
+        {
+            // Generate 1 test file per downloader.
+            var files = downloaders.Select(d => GenerateTestFile(uploader, d, testFileSize)).ToArray();
+
+            // Upload all the test files to the uploader.
+            var contentIds = files.Select(uploader.UploadFile).ToArray();
+
+            // Each downloader should retrieve its own test file.
+            for (var i = 0; i < downloaders.Length; i++)
+            {
+                var expectedFile = files[i];
+                var downloadedFile = downloaders[i].DownloadContent(contentIds[i], $"{expectedFile.Label}DOWNLOADED");
+
                 expectedFile.AssertIsEqual(downloadedFile);
-                return PeerConnectionState.Connection;
             }
-            catch
-            {
-                // Should an exception occur during the download or file-content assertion,
-                // We consider that as no-connection for the purpose of this test.
-                return PeerConnectionState.NoConnection;
-            }
-            finally
-            {
-                fileManager.PopFileSet();
-            }
-
-            // Should an exception occur during upload, then this try is inconclusive and we try again next loop.
         }
 
-        private TestFile DownloadFile(CodexAccess node, string contentId, string label)
-        {
-            var downloadedFile = fileManager.CreateEmptyTestFile(label);
-            using var downloadStream = File.OpenWrite(downloadedFile.Filename);
-            using var stream = node.DownloadFile(contentId);
-            stream.CopyTo(downloadStream);
-            return downloadedFile;
-        }
-
-        private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader)
+        private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader, ByteSize testFileSize)
         {
             var up = uploader.GetName().Replace("<", "").Replace(">", "");
             var down = downloader.GetName().Replace("<", "").Replace(">", "");
-            var label = $"~from:{up}-to:{down}~";
-            return fileManager.GenerateTestFile(testFileSize, label);
+            var label = $"FROM{up}TO{down}";
+            return test.GenerateTestFile(testFileSize, label);
         }
     }
 }

DistTestCore/Helpers/PeerFetchTestHelpers.cs

@@ -1,96 +0,0 @@
-using DistTestCore.Codex;
-using Logging;
-using NUnit.Framework;
-using static DistTestCore.Helpers.FullConnectivityHelper;
-
-namespace DistTestCore.Helpers
-{
-    public class PeerFetchTestHelpers : IFullConnectivityImplementation
-    {
-        private readonly FullConnectivityHelper helper;
-        private readonly BaseLog log;
-        private readonly IFileManager fileManager;
-        private readonly ByteSize testFileSize;
-        private readonly int expectedNumberOfBlocks;
-
-        public PeerFetchTestHelpers(BaseLog log, IFileManager fileManager)
-        {
-            helper = new FullConnectivityHelper(log, this);
-            testFileSize = 10.MB();
-            expectedNumberOfBlocks = 161;
-            this.log = log;
-            this.fileManager = fileManager;
-        }
-
-        public void AssertFullFetchInterconnectivity(IEnumerable<IOnlineCodexNode> nodes)
-        {
-            AssertFullFetchInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess));
-        }
-
-        public void AssertFullFetchInterconnectivity(IEnumerable<CodexAccess> nodes)
-        {
-            helper.AssertFullyConnected(nodes);
-        }
-
-        public string Description()
-        {
-            return "Fetch connectivity";
-        }
-
-        public string ValidateEntry(Entry entry, Entry[] allEntries)
-        {
-            return string.Empty;
-        }
-
-        public PeerConnectionState Check(Entry from, Entry to)
-        {
-            fileManager.PushFileSet();
-            var expectedFile = GenerateTestFile(from.Node, to.Node);
-
-            using var uploadStream = File.OpenRead(expectedFile.Filename);
-            var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(uploadStream));
-            var originalFetch = from.Node.DebugFetch(contentId);
-            Assert.That(Convert.ToInt32(originalFetch.numberOfBlocks), Is.EqualTo(expectedNumberOfBlocks));
-            Assert.That(originalFetch.originalBytes.Replace("'NByte", ""), Is.EqualTo("10485760"));
-
-            try
-            {
-                var fetchResponse = Stopwatch.Measure(log, "Fetch", () => to.Node.DebugFetch(contentId));
-                AssertEqual(originalFetch, fetchResponse);
-                return PeerConnectionState.Connection;
-            }
-            catch
-            {
-                return PeerConnectionState.NoConnection;
-            }
-            finally
-            {
-                fileManager.PopFileSet();
-            }
-        }
-
-        private void AssertEqual(CodexDebugFetchResponse expected, CodexDebugFetchResponse actual)
-        {
-            Assert.That(actual.originalBytes, Is.EqualTo(expected.originalBytes));
-            Assert.That(actual.blockSize, Is.EqualTo(expected.blockSize));
-            Assert.That(actual.numberOfBlocks, Is.EqualTo(expected.numberOfBlocks));
-            Assert.That(actual.version, Is.EqualTo(expected.version));
-            Assert.That(actual.hcodec, Is.EqualTo(expected.hcodec));
-            Assert.That(actual.codec, Is.EqualTo(expected.codec));
-            Assert.That(actual.@protected, Is.EqualTo(expected.@protected));
-            Assert.That(actual.blocks.Length, Is.EqualTo(expected.blocks.Length));
-            for (var i = 0; i < expected.blocks.Length; i++)
-            {
-                Assert.That(actual.blocks[i].cid, Is.EqualTo(expected.blocks[i].cid));
-            }
-        }
-
-        private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader)
-        {
-            var up = uploader.GetName().Replace("<", "").Replace(">", "");
-            var down = downloader.GetName().Replace("<", "").Replace(">", "");
-            var label = $"~from:{up}-to:{down}~";
-            return fileManager.GenerateTestFile(testFileSize, label);
-        }
-    }
-}

DistTestCore/Http.cs

@@ -1,6 +1,5 @@
 using Logging;
 using Newtonsoft.Json;
-using Serialization = Newtonsoft.Json.Serialization;
 using System.Net.Http.Headers;
 using System.Net.Http.Json;
 using Utils;
@@ -13,21 +12,14 @@ namespace DistTestCore
         private readonly ITimeSet timeSet;
         private readonly Address address;
         private readonly string baseUrl;
-        private readonly Action<HttpClient> onClientCreated;
         private readonly string? logAlias;
 
         public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
-            : this(log, timeSet, address, baseUrl, DoNothing, logAlias)
-        {
-        }
-
-        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
         {
             this.log = log;
             this.timeSet = timeSet;
             this.address = address;
             this.baseUrl = baseUrl;
-            this.onClientCreated = onClientCreated;
             this.logAlias = logAlias;
             if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
             if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
@@ -55,36 +47,23 @@ namespace DistTestCore
 
         public TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body)
         {
-            var response = PostJson(route, body);
-            var json = Time.Wait(response.Content.ReadAsStringAsync());
-            if (!response.IsSuccessStatusCode)
-            {
-                throw new HttpRequestException(json);
-            }
-            Log(GetUrl() + route, json);
+            var json = HttpPostJson(route, body);
             return TryJsonDeserialize<TResponse>(json);
         }
 
         public string HttpPostJson<TRequest>(string route, TRequest body)
-        {
-            var response = PostJson<TRequest>(route, body);
-            return Time.Wait(response.Content.ReadAsStringAsync());
-        }
-
-        public string HttpPostString(string route, string body)
         {
             return Retry(() =>
             {
                 using var client = GetClient();
                 var url = GetUrl() + route;
-                Log(url, body);
-                var content = new StringContent(body);
-                content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+                using var content = JsonContent.Create(body);
+                Log(url, JsonConvert.SerializeObject(body));
                 var result = Time.Wait(client.PostAsync(url, content));
                 var str = Time.Wait(result.Content.ReadAsStringAsync());
                 Log(url, str);
                 return str;
-            }, $"HTTP-POST-STRING: {route}");
+            }, $"HTTP-POST-JSON: {route}");
         }
 
         public string HttpPostStream(string route, Stream stream)
@@ -97,7 +76,7 @@ namespace DistTestCore
                 var content = new StreamContent(stream);
                 content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
                 var response = Time.Wait(client.PostAsync(url, content));
-                var str = Time.Wait(response.Content.ReadAsStringAsync());
+                var str =Time.Wait(response.Content.ReadAsStringAsync());
                 Log(url, str);
                 return str;
             }, $"HTTP-POST-STREAM: {route}");
@@ -116,43 +95,15 @@ namespace DistTestCore
 
         public T TryJsonDeserialize<T>(string json)
         {
-            var errors = new List<string>();
-            var deserialized = JsonConvert.DeserializeObject<T>(json, new JsonSerializerSettings()
+            try
             {
-                Error = delegate(object? sender, Serialization.ErrorEventArgs args)
-                {
-                    if (args.CurrentObject == args.ErrorContext.OriginalObject)
-                    {
-                        errors.Add($"""
-                                    Member: '{args.ErrorContext.Member?.ToString() ?? "<null>"}'
-                                    Path: {args.ErrorContext.Path}
-                                    Error: {args.ErrorContext.Error.Message}
-                                    """);
-                        args.ErrorContext.Handled = true;
-                    }
-                }
-            });
-            if (errors.Count() > 0)
-            {
-                throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}");
+                return JsonConvert.DeserializeObject<T>(json)!;
             }
-            else if (deserialized == null)
+            catch (Exception exception)
             {
-                throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null");
+                var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}";
+                throw new InvalidOperationException(msg, exception);
             }
-            return deserialized;
-        }
-
-        private HttpResponseMessage PostJson<TRequest>(string route, TRequest body)
-        {
-            return Retry(() =>
-            {
-                using var client = GetClient();
-                var url = GetUrl() + route;
-                using var content = JsonContent.Create(body);
-                Log(url, JsonConvert.SerializeObject(body));
-                return Time.Wait(client.PostAsync(url, content));
-            }, $"HTTP-POST-JSON: {route}");
         }
 
         private string GetUrl()
@@ -181,12 +132,7 @@ namespace DistTestCore
         {
             var client = new HttpClient();
             client.Timeout = timeSet.HttpCallTimeout();
-            onClientCreated(client);
             return client;
         }
-
-        private static void DoNothing(HttpClient client)
-        {
-        }
     }
 }

DistTestCore/Logs/DownloadedLog.cs

@@ -6,8 +6,6 @@ namespace DistTestCore.Logs
     public interface IDownloadedLog
     {
         void AssertLogContains(string expectedString);
-        string[] FindLinesThatContain(params string[] tags);
-        void DeleteFile();
     }
 
     public class DownloadedLog : IDownloadedLog
@@ -35,30 +33,5 @@ namespace DistTestCore.Logs
 
             Assert.Fail($"{owner} Unable to find string '{expectedString}' in CodexNode log file {logFile.FullFilename}");
         }
-
-        public string[] FindLinesThatContain(params string[] tags)
-        {
-            var result = new List<string>();
-            using var file = File.OpenRead(logFile.FullFilename);
-            using var streamReader = new StreamReader(file);
-
-            var line = streamReader.ReadLine();
-            while (line != null)
-            {
-                if (tags.All(line.Contains))
-                {
-                    result.Add(line);
-                }
-
-                line = streamReader.ReadLine();
-            }
-
-            return result.ToArray();
-        }
-
-        public void DeleteFile()
-        {
-            File.Delete(logFile.FullFilename);
-        }
     }
 }

DistTestCore/Marketplace/CodexContractsContainerRecipe.cs

@@ -2,15 +2,19 @@
 
 namespace DistTestCore.Marketplace
 {
-    public class CodexContractsContainerRecipe : DefaultContainerRecipe
+    public class CodexContractsContainerRecipe : ContainerRecipeFactory
     {
-        public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
-        public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
+        #if Arm64
+            public const string DockerImage = "emizzle/codex-contracts-deployment:latest";
+        #else
+            public const string DockerImage = "thatbenbierens/codex-contracts-deployment:nomint2";
+        #endif
+        public const string MarketplaceAddressFilename = "/usr/app/deployments/codexdisttestnetwork/Marketplace.json";
+        public const string MarketplaceArtifactFilename = "/usr/app/artifacts/contracts/Marketplace.sol/Marketplace.json";
 
-        public override string AppName => "codex-contracts";
-        public override string Image => "codexstorage/codex-contracts-eth:latest-dist-tests";
+        protected override string Image => DockerImage;
 
-        protected override void InitializeRecipe(StartupConfig startupConfig)
+        protected override void Initialize(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<CodexContractsContainerConfig>();
 

DistTestCore/Marketplace/CodexContractsStarter.cs

@@ -6,8 +6,8 @@ namespace DistTestCore.Marketplace
     public class CodexContractsStarter : BaseStarter
     {
 
-        public CodexContractsStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
+        public CodexContractsStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+            : base(lifecycle, workflowCreator)
         {
         }
 
@@ -15,7 +15,7 @@ namespace DistTestCore.Marketplace
         {
             LogStart("Deploying Codex Marketplace...");
 
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var workflow = workflowCreator.CreateWorkflow();
             var startupConfig = CreateStartupConfig(bootstrapNode.RunningContainers.Containers[0]);
 
             var containers = workflow.Start(1, Location.Unspecified, new CodexContractsContainerRecipe(), startupConfig);
@@ -25,7 +25,7 @@ namespace DistTestCore.Marketplace
             WaitUntil(() =>
             {
                 var logHandler = new ContractsReadyLogHandler(Debug);
-                workflow.DownloadContainerLog(container, logHandler, null);
+                workflow.DownloadContainerLog(container, logHandler);
                 return logHandler.Found;
             });
             Log("Contracts deployed. Extracting addresses...");
@@ -91,13 +91,8 @@ namespace DistTestCore.Marketplace
         {
             debug(line);
             if (line.Contains(RequiredCompiledString)) SeenCompileString = true;
-            if (line.Contains(ReadyString))
-            {
-                if (!SeenCompileString) throw new Exception("CodexContracts deployment failed. " +
-                    "Solidity files not compiled before process exited.");
 
-                Found = true;
-            }
+            if (SeenCompileString && line.Contains(ReadyString)) Found = true;
         }
     }
 }

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -80,7 +80,7 @@ namespace DistTestCore.Marketplace
         private string FetchPubKey()
         {
             var enodeFinder = new PubKeyFinder(s => log.Debug(s));
-            workflow.DownloadContainerLog(container, enodeFinder, null);
+            workflow.DownloadContainerLog(container, enodeFinder);
             return enodeFinder.GetPubKey();
         }
 

DistTestCore/Marketplace/GethBootstrapNodeStarter.cs

@@ -4,8 +4,8 @@ namespace DistTestCore.Marketplace
 {
     public class GethBootstrapNodeStarter : BaseStarter
     {
-        public GethBootstrapNodeStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
+        public GethBootstrapNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+            : base(lifecycle, workflowCreator)
         {
         }
 
@@ -14,7 +14,7 @@ namespace DistTestCore.Marketplace
             LogStart("Starting Geth bootstrap node...");
             var startupConfig = CreateBootstrapStartupConfig();
             
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var workflow = workflowCreator.CreateWorkflow();
             var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), startupConfig);
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Geth bootstrap node to be created. Test infra failure.");
             var bootstrapContainer = containers.Containers[0];

DistTestCore/Marketplace/GethCompanionNodeStarter.cs

@@ -7,8 +7,8 @@ namespace DistTestCore.Marketplace
     {
         private int companionAccountIndex = 0;
 
-        public GethCompanionNodeStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
+        public GethCompanionNodeStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+            : base(lifecycle, workflowCreator)
         {
         }
 
@@ -18,7 +18,7 @@ namespace DistTestCore.Marketplace
 
             var config = CreateCompanionNodeStartupConfig(marketplace.Bootstrap, codexSetup.NumberOfNodes);
 
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var workflow = workflowCreator.CreateWorkflow();
             var containers = workflow.Start(1, Location.Unspecified, new GethContainerRecipe(), CreateStartupConfig(config));
             if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected one Geth companion node to be created. Test infra failure.");
             var container = containers.Containers[0];

DistTestCore/Marketplace/GethContainerRecipe.cs

@@ -2,18 +2,23 @@
 
 namespace DistTestCore.Marketplace
 {
-    public class GethContainerRecipe : DefaultContainerRecipe
+    public class GethContainerRecipe : ContainerRecipeFactory
     {
-        private const string defaultArgs = "--ipcdisable --syncmode full";
+        #if Arm64
+            public const string DockerImage = "emizzle/geth-confenv:latest";
+        #else
+            public const string DockerImage = "thatbenbierens/geth-confenv:onethousand";
+        #endif
 
         public const string HttpPortTag = "http_port";
         public const string DiscoveryPortTag = "disc_port";
+        private const string defaultArgs = "--ipcdisable --syncmode full";
+
         public const string AccountsFilename = "accounts.csv";
 
-        public override string AppName => "geth";
-        public override string Image => "codexstorage/dist-tests-geth:latest";
+        protected override string Image => DockerImage;
 
-        protected override void InitializeRecipe(StartupConfig startupConfig)
+        protected override void Initialize(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<GethStartupConfig>();
 

DistTestCore/Marketplace/MarketplaceAccess.cs

@@ -1,7 +1,5 @@
 using DistTestCore.Codex;
 using DistTestCore.Helpers;
-using Logging;
-using Newtonsoft.Json;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
 using System.Numerics;
@@ -12,7 +10,7 @@ namespace DistTestCore.Marketplace
     public interface IMarketplaceAccess
     {
         string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan maxDuration);
-        StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
+        string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration);
         void AssertThatBalance(IResolveConstraint constraint, string message = "");
         TestToken GetBalance();
     }
@@ -32,7 +30,7 @@ namespace DistTestCore.Marketplace
             this.codexAccess = codexAccess;
         }
 
-        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        public string RequestStorage(ContentId contentId, TestToken pricePerSlotPerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
         {
             var request = new CodexSalesRequestStorageRequest
             {
@@ -59,9 +57,9 @@ namespace DistTestCore.Marketplace
                 throw new InvalidOperationException(response);
             }
 
-            Log($"Storage requested successfully. PurchaseId: '{response}'.");
+            Log($"Storage requested successfully. PurchaseId: {response}");
 
-            return new StoragePurchaseContract(lifecycle.Log, codexAccess, response, duration);
+            return response;
         }
 
         public string MakeStorageAvailable(ByteSize totalSpace, TestToken minPriceForTotalSpace, TestToken maxCollateral, TimeSpan maxDuration)
@@ -123,10 +121,10 @@ namespace DistTestCore.Marketplace
 
     public class MarketplaceUnavailable : IMarketplaceAccess
     {
-        public StoragePurchaseContract RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
+        public string RequestStorage(ContentId contentId, TestToken pricePerBytePerSecond, TestToken requiredCollateral, uint minRequiredNumberOfNodes, int proofProbability, TimeSpan duration)
         {
             Unavailable();
-            return null!;
+            return string.Empty;
         }
 
         public string MakeStorageAvailable(ByteSize size, TestToken minPricePerBytePerSecond, TestToken maxCollateral, TimeSpan duration)
@@ -152,92 +150,4 @@ namespace DistTestCore.Marketplace
             throw new InvalidOperationException();
         }
     }
-
-    public class StoragePurchaseContract
-    {
-        private readonly BaseLog log;
-        private readonly CodexAccess codexAccess;
-        private DateTime? contractStartUtc;
-
-        public StoragePurchaseContract(BaseLog log, CodexAccess codexAccess, string purchaseId, TimeSpan contractDuration)
-        {
-            this.log = log;
-            this.codexAccess = codexAccess;
-            PurchaseId = purchaseId;
-            ContractDuration = contractDuration;
-        }
-
-        public string PurchaseId { get; }
-        public TimeSpan ContractDuration { get; }
-
-        public void WaitForStorageContractStarted()
-        {
-            WaitForStorageContractStarted(TimeSpan.FromSeconds(30));
-        }
-
-        public void WaitForStorageContractFinished()
-        {
-            if (!contractStartUtc.HasValue)
-            {
-                WaitForStorageContractStarted();
-            }
-            var gracePeriod = TimeSpan.FromSeconds(10);
-            var currentContractTime = DateTime.UtcNow - contractStartUtc!.Value;
-            var timeout = (ContractDuration - currentContractTime) + gracePeriod;
-            WaitForStorageContractState(timeout, "finished");
-        }
-
-        /// <summary>
-        /// Wait for contract to start. Max timeout depends on contract filesize. Allows more time for larger files.
-        /// </summary>
-        public void WaitForStorageContractStarted(ByteSize contractFileSize)
-        {
-            var filesizeInMb = contractFileSize.SizeInBytes / (1024 * 1024);
-            var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0);
-
-            WaitForStorageContractStarted(maxWaitTime);
-        }
-
-        public void WaitForStorageContractStarted(TimeSpan timeout)
-        {
-            WaitForStorageContractState(timeout, "started");
-            contractStartUtc = DateTime.UtcNow;
-        }
-
-        private void WaitForStorageContractState(TimeSpan timeout, string desiredState)
-        {
-            var lastState = "";
-            var waitStart = DateTime.UtcNow;
-            
-            log.Log($"Waiting for {Time.FormatDuration(timeout)} for contract '{PurchaseId}' to reach state '{desiredState}'.");
-            while (lastState != desiredState)
-            {
-                var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId);
-                var statusJson = JsonConvert.SerializeObject(purchaseStatus);
-                if (purchaseStatus != null && purchaseStatus.state != lastState)
-                {
-                    lastState = purchaseStatus.state;
-                    log.Debug("Purchase status: " + statusJson);
-                }
-
-                Thread.Sleep(1000);
-
-                if (lastState == "errored")
-                {
-                    Assert.Fail("Contract errored: " + statusJson);
-                }
-
-                if (DateTime.UtcNow - waitStart > timeout)
-                {
-                    Assert.Fail($"Contract did not reach '{desiredState}' within timeout. {statusJson}");
-                }
-            }
-            log.Log($"Contract '{desiredState}'.");
-        }
-
-        public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
-        {
-            return codexAccess.GetPurchaseStatus(purchaseId);
-        }
-    }
 }

DistTestCore/Metrics/GrafanaContainerRecipe.cs

@@ -1,25 +0,0 @@
-using KubernetesWorkflow;
-
-namespace DistTestCore.Metrics
-{
-    public class GrafanaContainerRecipe : DefaultContainerRecipe
-    {
-        public override string AppName => "grafana";
-        public override string Image => "grafana/grafana-oss:10.0.3";
-
-        public const string DefaultAdminUser = "adminium";
-        public const string DefaultAdminPassword = "passwordium";
-
-        protected override void InitializeRecipe(StartupConfig startupConfig)
-        {
-            AddExposedPort(3000);
-
-            AddEnvVar("GF_AUTH_ANONYMOUS_ENABLED", "true");
-            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_NAME", "Main Org.");
-            AddEnvVar("GF_AUTH_ANONYMOUS_ORG_ROLE", "Editor");
-
-            AddEnvVar("GF_SECURITY_ADMIN_USER", DefaultAdminUser);
-            AddEnvVar("GF_SECURITY_ADMIN_PASSWORD", DefaultAdminPassword);
-        }
-    }
-}

DistTestCore/Metrics/MetricsMode.cs

@@ -1,9 +0,0 @@
-namespace DistTestCore.Metrics
-{
-    public enum MetricsMode
-    {
-        None,
-        Record,
-        Dashboard
-    }
-}

DistTestCore/Metrics/PrometheusContainerRecipe.cs

@@ -2,12 +2,13 @@
 
 namespace DistTestCore.Metrics
 {
-    public class PrometheusContainerRecipe : DefaultContainerRecipe
+    public class PrometheusContainerRecipe : ContainerRecipeFactory
     {
-        public override string AppName => "prometheus";
-        public override string Image => "codexstorage/dist-tests-prometheus:latest";
+        public const string DockerImage = "thatbenbierens/prometheus-envconf:latest";
 
-        protected override void InitializeRecipe(StartupConfig startupConfig)
+        protected override string Image => DockerImage;
+
+        protected override void Initialize(StartupConfig startupConfig)
         {
             var config = startupConfig.Get<PrometheusStartupConfig>();
 

DistTestCore/OnlineCodexNode.cs

@@ -4,7 +4,6 @@ using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
 using Logging;
 using NUnit.Framework;
-using Utils;
 
 namespace DistTestCore
 {
@@ -16,10 +15,9 @@ namespace DistTestCore
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(IOnlineCodexNode node);
-        IDownloadedLog DownloadLog(int? tailLines = null);
+        IDownloadedLog DownloadLog();
         IMetricsAccess Metrics { get; }
         IMarketplaceAccess Marketplace { get; }
-        CodexDebugVersionResponse Version { get; }
         ICodexSetup BringOffline();
     }
 
@@ -36,14 +34,12 @@ namespace DistTestCore
             Group = group;
             Metrics = metricsAccess;
             Marketplace = marketplaceAccess;
-            Version = new CodexDebugVersionResponse();
         }
 
         public CodexAccess CodexAccess { get; }
         public CodexNodeGroup Group { get; }
         public IMetricsAccess Metrics { get; }
         public IMarketplaceAccess Marketplace { get; }
-        public CodexDebugVersionResponse Version { get; private set; }
 
         public string GetName()
         {
@@ -68,7 +64,6 @@ namespace DistTestCore
             using var fileStream = File.OpenRead(file.Filename);
 
             var logMessage = $"Uploading file {file.Describe()}...";
-            Log(logMessage);
             var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
             {
                 return CodexAccess.UploadFile(fileStream);
@@ -77,6 +72,9 @@ namespace DistTestCore
             if (string.IsNullOrEmpty(response)) Assert.Fail("Received empty response.");
             if (response.StartsWith(UploadFailedMessage)) Assert.Fail("Node failed to store block.");
 
+            var logReplacement = $"(CID:{file.Describe()})";
+            Log($"ContentId '{response}' is {logReplacement}");
+            lifecycle.Log.AddStringReplace(response, logReplacement);
             Log($"Uploaded file. Received contentId: '{response}'.");
             return new ContentId(response);
         }
@@ -84,7 +82,6 @@ namespace DistTestCore
         public TestFile? DownloadContent(ContentId contentId, string fileLabel = "")
         {
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
-            Log(logMessage);
             var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel);
             Stopwatch.Measure(lifecycle.Log, logMessage, () => DownloadToFile(contentId.Id, file));
             Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
@@ -103,9 +100,9 @@ namespace DistTestCore
             Log($"Successfully connected to peer {peer.GetName()}.");
         }
 
-        public IDownloadedLog DownloadLog(int? tailLines = null)
+        public IDownloadedLog DownloadLog()
         {
-            return lifecycle.DownloadLog(CodexAccess.Container, tailLines);
+            return lifecycle.DownloadLog(CodexAccess.Container);
         }
 
         public ICodexSetup BringOffline()
@@ -117,30 +114,19 @@ namespace DistTestCore
             return Group.BringOffline();
         }
 
-        public void EnsureOnlineGetVersionResponse()
-        {
-            var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
-            var nodePeerId = debugInfo.id;
-            var nodeName = CodexAccess.Container.Name;
-
-            if (!debugInfo.codex.IsValid())
-            {
-                throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}");
-            }
-
-            lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
-            lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
-            Version = debugInfo.codex;
-        }
-
         private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
         {
             var multiAddress = peerInfo.addrs.First();
             // Todo: Is there a case where First address in list is not the way?
 
+            if (Group == peer.Group)
+            {
+                return multiAddress;
+            }
+
             // The peer we want to connect is in a different pod.
             // We must replace the default IP with the pod IP in the multiAddress.
-            return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip);
+            return multiAddress.Replace("0.0.0.0", peer.Group.Containers.RunningPod.PodInfo.Ip);
         }
 
         private void DownloadToFile(string contentId, TestFile file)

DistTestCore/PrometheusStarter.cs

@@ -7,21 +7,23 @@ namespace DistTestCore
 {
     public class PrometheusStarter : BaseStarter
     {
-        public PrometheusStarter(TestLifecycle lifecycle)
-            : base(lifecycle)
+        public PrometheusStarter(TestLifecycle lifecycle, WorkflowCreator workflowCreator)
+            : base(lifecycle, workflowCreator)
         {
         }
 
-        public RunningContainers CollectMetricsFor(RunningContainers[] containers)
+        public RunningContainers CollectMetricsFor(RunningContainers containers)
         {
             LogStart($"Starting metrics server for {containers.Describe()}");
             var startupConfig = new StartupConfig();
-            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers())));
+            startupConfig.Add(new PrometheusStartupConfig(GeneratePrometheusConfig(containers.Containers)));
 
-            var workflow = lifecycle.WorkflowCreator.CreateWorkflow();
+            var workflow = workflowCreator.CreateWorkflow();
             var runningContainers = workflow.Start(1, Location.Unspecified, new PrometheusContainerRecipe(), startupConfig);
             if (runningContainers.Containers.Length != 1) throw new InvalidOperationException("Expected only 1 Prometheus container to be created.");
 
+            LogEnd("Metrics server started.");
+
             return runningContainers;
         }
 
@@ -29,7 +31,7 @@ namespace DistTestCore
         {
             var config = "";
             config += "global:\n";
-            config += "  scrape_interval: 10s\n";
+            config += "  scrape_interval: 30s\n";
             config += "  scrape_timeout: 10s\n";
             config += "\n";
             config += "scrape_configs:\n";

DistTestCore/TestLifecycle.cs

@@ -1,7 +1,4 @@
-using DistTestCore.Codex;
-using DistTestCore.Logs;
-using DistTestCore.Marketplace;
-using DistTestCore.Metrics;
+using DistTestCore.Logs;
 using KubernetesWorkflow;
 using Logging;
 using Utils;
@@ -10,37 +7,33 @@ namespace DistTestCore
 {
     public class TestLifecycle
     {
-        private readonly DateTime testStart;
+        private DateTime testStart = DateTime.MinValue;
 
-        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
+        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet)
+            : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)))
+        {
+        }
+
+        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
         {
             Log = log;
             Configuration = configuration;
             TimeSet = timeSet;
 
-            WorkflowCreator = new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet), testNamespace);
-
             FileManager = new FileManager(Log, configuration);
-            CodexStarter = new CodexStarter(this);
-            PrometheusStarter = new PrometheusStarter(this);
-            GrafanaStarter = new GrafanaStarter(this);
-            GethStarter = new GethStarter(this);
+            CodexStarter = new CodexStarter(this, workflowCreator);
+            PrometheusStarter = new PrometheusStarter(this, workflowCreator);
+            GethStarter = new GethStarter(this, workflowCreator);
             testStart = DateTime.UtcNow;
-            CodexVersion = null;
-
-            Log.WriteLogTag();
         }
 
         public BaseLog Log { get; }
         public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
-        public WorkflowCreator WorkflowCreator { get; }
         public FileManager FileManager { get; }
         public CodexStarter CodexStarter { get; }
         public PrometheusStarter PrometheusStarter { get; }
-        public GrafanaStarter GrafanaStarter { get; }
         public GethStarter GethStarter { get; }
-        public CodexDebugVersionResponse? CodexVersion { get; private set; }
 
         public void DeleteAllResources()
         {
@@ -48,14 +41,14 @@ namespace DistTestCore
             FileManager.DeleteAllTestFiles();
         }
 
-        public IDownloadedLog DownloadLog(RunningContainer container, int? tailLines = null)
+        public IDownloadedLog DownloadLog(RunningContainer container)
         {
             var subFile = Log.CreateSubfile();
             var description = container.Name;
             var handler = new LogDownloadHandler(container, description, subFile);
 
             Log.Log($"Downloading logs for {description} to file '{subFile.FullFilename}'");
-            CodexStarter.DownloadLog(container, handler, tailLines);
+            CodexStarter.DownloadLog(container, handler);
 
             return new DownloadedLog(subFile, description);
         }
@@ -65,29 +58,5 @@ namespace DistTestCore
             var testDuration = DateTime.UtcNow - testStart;
             return Time.FormatDuration(testDuration);
         }
-
-        public void SetCodexVersion(CodexDebugVersionResponse version)
-        {
-            if (CodexVersion == null) CodexVersion = version;
-        }
-
-        public ApplicationIds GetApplicationIds()
-        {
-            return new ApplicationIds(
-                codexId: GetCodexId(),
-                gethId: new GethContainerRecipe().Image,
-                prometheusId: new PrometheusContainerRecipe().Image,
-                codexContractsId: new CodexContractsContainerRecipe().Image,
-                grafanaId: new GrafanaContainerRecipe().Image
-            );
-        }
-
-        private string GetCodexId()
-        {
-            var v = CodexVersion;
-            if (v == null) return new CodexContainerRecipe().Image;
-            if (v.version != "untagged build") return v.version;
-            return v.revision;
-        }
     }
 }

DistTestCore/Timing.cs

@@ -21,7 +21,7 @@ namespace DistTestCore
     {
         public TimeSpan HttpCallTimeout()
         {
-            return TimeSpan.FromMinutes(5);
+            return TimeSpan.FromSeconds(10);
         }
 
         public TimeSpan HttpCallRetryTime()
@@ -36,12 +36,12 @@ namespace DistTestCore
 
         public TimeSpan WaitForK8sServiceDelay()
         {
-            return TimeSpan.FromSeconds(10);
+            return TimeSpan.FromSeconds(1);
         }
 
         public TimeSpan K8sOperationTimeout()
         {
-            return TimeSpan.FromMinutes(30);
+            return TimeSpan.FromMinutes(1);
         }
 
         public TimeSpan WaitForMetricTimeout()

DistTestCore/cs-codex-dist-tests.code-workspace

@@ -0,0 +1,11 @@
+{
+	"folders": [
+		{
+			"path": ".."
+		},
+		{
+			"path": "../../../CodexTestLogs"
+		}
+	],
+	"settings": {}
+}

KubernetesWorkflow/CommandRunner.cs

@@ -1,4 +1,5 @@
-using Utils;
+using k8s;
+using Utils;
 
 namespace KubernetesWorkflow
 {

KubernetesWorkflow/ContainerRecipe.cs

@@ -2,15 +2,13 @@
 {
     public class ContainerRecipe
     {
-        public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, PodLabels podLabels, PodAnnotations podAnnotations, object[] additionals)
+        public ContainerRecipe(int number, string image, Port[] exposedPorts, Port[] internalPorts, EnvVar[] envVars, object[] additionals)
         {
             Number = number;
             Image = image;
             ExposedPorts = exposedPorts;
             InternalPorts = internalPorts;
             EnvVars = envVars;
-            PodLabels = podLabels;
-            PodAnnotations = podAnnotations;
             Additionals = additionals;
         }
 
@@ -20,8 +18,6 @@
         public Port[] ExposedPorts { get; }
         public Port[] InternalPorts { get; }
         public EnvVar[] EnvVars { get; }
-        public PodLabels PodLabels { get; }
-        public PodAnnotations PodAnnotations { get; }
         public object[] Additionals { get; }
 
         public Port GetPortByTag(string tag)

KubernetesWorkflow/ContainerRecipeFactory.cs

@@ -5,8 +5,6 @@
         private readonly List<Port> exposedPorts = new List<Port>();
         private readonly List<Port> internalPorts = new List<Port>();
         private readonly List<EnvVar> envVars = new List<EnvVar>();
-        private readonly PodLabels podLabels = new PodLabels();
-        private readonly PodAnnotations podAnnotations = new PodAnnotations();
         private readonly List<object> additionals = new List<object>();
         private RecipeComponentFactory factory = null!;
 
@@ -18,27 +16,18 @@
 
             Initialize(config);
 
-            var recipe = new ContainerRecipe(containerNumber, Image,
-                exposedPorts.ToArray(),
-                internalPorts.ToArray(), 
-                envVars.ToArray(), 
-                podLabels.Clone(),
-                podAnnotations.Clone(),
-                additionals.ToArray());
+            var recipe = new ContainerRecipe(containerNumber, Image, exposedPorts.ToArray(), internalPorts.ToArray(), envVars.ToArray(), additionals.ToArray());
 
             exposedPorts.Clear();
             internalPorts.Clear();
             envVars.Clear();
-            podLabels.Clear();
-            podAnnotations.Clear();
             additionals.Clear();
             this.factory = null!;
 
             return recipe;
         }
 
-        public abstract string AppName { get; }
-        public abstract string Image { get; }
+        protected abstract string Image { get; }
         protected int ContainerNumber { get; private set; } = 0;
         protected int Index { get; private set; } = 0;
         protected abstract void Initialize(StartupConfig config);
@@ -50,13 +39,6 @@
             return p;
         }
 
-        protected Port AddExposedPort(int number, string tag = "")
-        {
-            var p = factory.CreatePort(number, tag);
-            exposedPorts.Add(p);
-            return p;
-        }
-
         protected Port AddInternalPort(string tag = "")
         {
             var p = factory.CreatePort(tag);
@@ -84,16 +66,6 @@
             envVars.Add(factory.CreateEnvVar(name, value.Number));
         }
 
-        protected void AddPodLabel(string name, string value)
-        {
-            podLabels.Add(name, value);
-        }
-
-        protected void AddPodAnnotation(string name, string value)
-        {
-            podAnnotations.Add(name, value);
-        }
-
         protected void Additional(object userData)
         {
             additionals.Add(userData);

KubernetesWorkflow/CrashWatcher.cs

@@ -1,95 +0,0 @@
-using k8s;
-using Logging;
-
-namespace KubernetesWorkflow
-{
-    public class CrashWatcher
-    {
-        private readonly BaseLog log;
-        private readonly KubernetesClientConfiguration config;
-        private readonly string k8sNamespace;
-        private readonly RunningContainer container;
-        private ILogHandler? logHandler;
-        private CancellationTokenSource cts;
-        private Task? worker;
-        private Exception? workerException;
-
-        public CrashWatcher(BaseLog log, KubernetesClientConfiguration config, string k8sNamespace, RunningContainer container)
-        {
-            this.log = log;
-            this.config = config;
-            this.k8sNamespace = k8sNamespace;
-            this.container = container;
-            cts = new CancellationTokenSource();
-        }
-
-        public void Start(ILogHandler logHandler)
-        {
-            if (worker != null) throw new InvalidOperationException();
-
-            this.logHandler = logHandler;
-            cts = new CancellationTokenSource();
-            worker = Task.Run(Worker);
-        }
-
-        public void Stop()
-        {
-            if (worker == null) throw new InvalidOperationException();
-
-            cts.Cancel();
-            worker.Wait();
-            worker = null;
-
-            if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException);
-        }
-
-        public bool HasContainerCrashed()
-        {
-            using var client = new Kubernetes(config);
-            return HasContainerBeenRestarted(client, container.Pod.PodInfo.Name);
-        }
-
-        private void Worker()
-        {
-            try
-            {
-                MonitorContainer(cts.Token);
-            }
-            catch (Exception ex)
-            {
-                workerException = ex;
-            }
-        }
-
-        private void MonitorContainer(CancellationToken token)
-        {
-            using var client = new Kubernetes(config);
-            while (!token.IsCancellationRequested)
-            {
-                token.WaitHandle.WaitOne(TimeSpan.FromSeconds(1));
-
-                var pod = container.Pod;
-                var recipe = container.Recipe;
-                var podName = pod.PodInfo.Name;
-                if (HasContainerBeenRestarted(client, podName))
-                {
-                    DownloadCrashedContainerLogs(client, podName, recipe);
-                    return;
-                }
-            }
-        }
-
-        private bool HasContainerBeenRestarted(Kubernetes client, string podName)
-        {
-            var podInfo = client.ReadNamespacedPod(podName, k8sNamespace);
-            return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0);
-        }
-
-        private void DownloadCrashedContainerLogs(Kubernetes client, string podName, ContainerRecipe recipe)
-        {
-            log.Log("Pod crash detected for " + container.Name);
-            using var stream = client.ReadNamespacedPodLog(podName, k8sNamespace, recipe.Name, previous: true);
-            logHandler!.Log(stream);
-        }
-    }
-}

KubernetesWorkflow/K8sCluster.cs

@@ -8,7 +8,7 @@ namespace KubernetesWorkflow
         {
             Configuration = configuration;
         }
-
+       
         public Configuration Configuration { get; }
         public string HostAddress { get; private set; } = string.Empty;
         public K8sNodeLabel[] AvailableK8sNodes { get; set; } = new K8sNodeLabel[0];

KubernetesWorkflow/K8sController.cs

@@ -51,10 +51,10 @@ namespace KubernetesWorkflow
             WaitUntilPodOffline(pod.PodInfo.Name);
         }
 
-        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler, int? tailLines)
+        public void DownloadPodLog(RunningPod pod, ContainerRecipe recipe, ILogHandler logHandler)
         {
             log.Debug();
-            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name, tailLines: tailLines));
+            using var stream = client.Run(c => c.ReadNamespacedPodLog(pod.PodInfo.Name, K8sTestNamespace, recipe.Name));
             logHandler.Log(stream);
         }
 
@@ -219,19 +219,6 @@ namespace KubernetesWorkflow
                                         }
                                     }
                                 }
-                            },
-                            new V1NetworkPolicyIngressRule
-                            {
-                                FromProperty = new List<V1NetworkPolicyPeer>
-                                {
-                                    new V1NetworkPolicyPeer
-                                    {
-                                        NamespaceSelector = new V1LabelSelector
-                                        {
-                                            MatchLabels = GetPrometheusNamespaceSelector()
-                                        }
-                                    }
-                                }
                             }
                         },
                         Egress = new List<V1NetworkPolicyEgressRule>
@@ -285,7 +272,7 @@ namespace KubernetesWorkflow
                                     {
                                         IpBlock = new V1IPBlock
                                         {
-                                            Cidr = "0.0.0.0/0"
+                                          Cidr = "0.0.0.0/0"
                                         }
                                     }
                                 },
@@ -327,20 +314,19 @@ namespace KubernetesWorkflow
             var deploymentSpec = new V1Deployment
             {
                 ApiVersion = "apps/v1",
-                Metadata = CreateDeploymentMetadata(containerRecipes),
+                Metadata = CreateDeploymentMetadata(),
                 Spec = new V1DeploymentSpec
                 {
                     Replicas = 1,
                     Selector = new V1LabelSelector
                     {
-                        MatchLabels = GetSelector(containerRecipes)
+                        MatchLabels = GetSelector()
                     },
                     Template = new V1PodTemplateSpec
                     {
                         Metadata = new V1ObjectMeta
                         {
-                            Labels = GetSelector(containerRecipes),
-                            Annotations = GetAnnotations(containerRecipes)
+                            Labels = GetSelector()
                         },
                         Spec = new V1PodSpec
                         {
@@ -374,9 +360,9 @@ namespace KubernetesWorkflow
             };
         }
 
-        private IDictionary<string, string> GetSelector(ContainerRecipe[] containerRecipes)
+        private IDictionary<string, string> GetSelector()
         {
-            return containerRecipes.First().PodLabels.GetLabels();
+            return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + workflowNumberSource.WorkflowNumber } };
         }
 
         private IDictionary<string, string> GetRunnerNamespaceSelector()
@@ -384,24 +370,13 @@ namespace KubernetesWorkflow
             return new Dictionary<string, string> { { "kubernetes.io/metadata.name", "default" } };
         }
 
-        private IDictionary<string, string> GetPrometheusNamespaceSelector()
-        {
-            return new Dictionary<string, string> { { "kubernetes.io/metadata.name", "monitoring" } };
-        }
-
-        private IDictionary<string, string> GetAnnotations(ContainerRecipe[] containerRecipes)
-        {
-            return containerRecipes.First().PodAnnotations.GetAnnotations();
-        }
-
-        private V1ObjectMeta CreateDeploymentMetadata(ContainerRecipe[] containerRecipes)
+        private V1ObjectMeta CreateDeploymentMetadata()
         {
             return new V1ObjectMeta
             {
                 Name = "deploy-" + workflowNumberSource.WorkflowNumber,
                 NamespaceProperty = K8sTestNamespace,
-                Labels = GetSelector(containerRecipes),
-                Annotations = GetAnnotations(containerRecipes)
+                Labels = GetSelector()
             };
         }
 
@@ -448,7 +423,7 @@ namespace KubernetesWorkflow
             return new V1ContainerPort
             {
                 Name = GetNameForPort(recipe, port),
-                ContainerPort = port.Number
+                ContainerPort = port.Number 
             };
         }
 
@@ -481,7 +456,7 @@ namespace KubernetesWorkflow
                 Spec = new V1ServiceSpec
                 {
                     Type = "NodePort",
-                    Selector = GetSelector(containerRecipes),
+                    Selector = GetSelector(),
                     Ports = ports
                 }
             };
@@ -627,11 +602,6 @@ namespace KubernetesWorkflow
 
         #endregion
 
-        public CrashWatcher CreateCrashWatcher(RunningContainer container)
-        {
-            return new CrashWatcher(log, cluster.GetK8sClientConfig(), K8sTestNamespace, container);
-        }
-
         private PodInfo FetchNewPod()
         {
             var pods = client.Run(c => c.ListNamespacedPod(K8sTestNamespace)).Items;

KubernetesWorkflow/PodAnnotations.cs

@@ -1,29 +0,0 @@
-namespace KubernetesWorkflow
-{
-    public class PodAnnotations
-    {
-        private readonly Dictionary<string, string> annotations = new Dictionary<string, string>();
-
-        public void Add(string key, string value)
-        {
-            annotations.Add(key, value);
-        }
-
-        public PodAnnotations Clone()
-        {
-            var result = new PodAnnotations();
-            foreach (var entry in annotations) result.Add(entry.Key, entry.Value);
-            return result;
-        }
-
-        public void Clear()
-        {
-            annotations.Clear();
-        }
-
-        internal Dictionary<string, string> GetAnnotations()
-        {
-            return annotations;
-        }
-    }
-}

KubernetesWorkflow/PodLabels.cs

@@ -1,42 +0,0 @@
-namespace KubernetesWorkflow
-{
-    public class PodLabels
-    {
-        private readonly Dictionary<string, string> labels = new Dictionary<string, string>();
-
-        public void Add(string key, string value)
-        {
-            labels.Add(key, Format(value));
-        }
-
-        public PodLabels Clone()
-        {
-            var result = new PodLabels();
-            foreach (var entry in labels) result.Add(entry.Key, entry.Value);
-            return result;
-        }
-
-        public void Clear()
-        {
-            labels.Clear();
-        }
-
-        private static string Format(string s)
-        {
-            var result = s.ToLowerInvariant()
-                .Replace(":", "-")
-                .Replace("/", "-")
-                .Replace("\\", "-")
-                .Replace("[", "-")
-                .Replace("]", "-")
-                .Replace(",", "-");
-
-            return result.Trim('-');
-        }
-
-        internal Dictionary<string, string> GetLabels()
-        {
-            return labels;
-        }
-    }
-}

KubernetesWorkflow/RecipeComponentFactory.cs

@@ -7,11 +7,6 @@ namespace KubernetesWorkflow
     {
         private NumberSource portNumberSource = new NumberSource(8080);
 
-        public Port CreatePort(int number, string tag)
-        {
-            return new Port(number, tag);
-        }
-
         public Port CreatePort(string tag)
         {
             return new Port(portNumberSource.GetNextNumber(), tag);

KubernetesWorkflow/RunningContainers.cs

@@ -1,5 +1,4 @@
-using Newtonsoft.Json;
-using Utils;
+using Utils;
 
 namespace KubernetesWorkflow
 {
@@ -40,21 +39,5 @@ namespace KubernetesWorkflow
         public Port[] ServicePorts { get; }
         public Address ClusterExternalAddress { get; }
         public Address ClusterInternalAddress { get; }
-
-        [JsonIgnore]
-        public CrashWatcher? CrashWatcher { get; set; }
-    }
-
-    public static class RunningContainersExtensions
-    {
-        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
-        {
-            return runningContainers.SelectMany(c => c.Containers).ToArray();
-        }
-
-        public static string Describe(this RunningContainers[] runningContainers)
-        {
-            return string.Join(",", runningContainers.Select(c => c.Describe()));
-        }
     }
 }

KubernetesWorkflow/StartupWorkflow.cs

@@ -33,11 +33,6 @@ namespace KubernetesWorkflow
             });
         }
 
-        public CrashWatcher CreateCrashWatcher(RunningContainer container)
-        {
-            return K8s(controller => controller.CreateCrashWatcher(container));
-        }
-
         public void Stop(RunningContainers runningContainers)
         {
             K8s(controller =>
@@ -46,11 +41,11 @@ namespace KubernetesWorkflow
             });
         }
 
-        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler, int? tailLines)
+        public void DownloadContainerLog(RunningContainer container, ILogHandler logHandler)
         {
             K8s(controller =>
             {
-                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler, tailLines);
+                controller.DownloadPodLog(container.Pod, container.Recipe, logHandler);
             });
         }
 

KubernetesWorkflow/WorkflowCreator.cs

@@ -12,11 +12,16 @@ namespace KubernetesWorkflow
         private readonly BaseLog log;
         private readonly string testNamespace;
 
-        public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespace)
+        public WorkflowCreator(BaseLog log, Configuration configuration)
+            : this(log, configuration, Guid.NewGuid().ToString().ToLowerInvariant())
+        {
+        }
+
+        public WorkflowCreator(BaseLog log, Configuration configuration, string testNamespacePostfix)
         {
             cluster = new K8sCluster(configuration);
             this.log = log;
-            this.testNamespace = testNamespace.ToLowerInvariant();
+            testNamespace = testNamespacePostfix;
         }
 
         public StartupWorkflow CreateWorkflow()

Logging/ApplicationIds.cs

@@ -1,20 +0,0 @@
-namespace Logging
-{
-    public class ApplicationIds
-    {
-        public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId, string grafanaId)
-        {
-            CodexId = codexId;
-            GethId = gethId;
-            PrometheusId = prometheusId;
-            CodexContractsId = codexContractsId;
-            GrafanaId = grafanaId;
-        }
-
-        public string CodexId { get; }
-        public string GethId { get; }
-        public string PrometheusId { get; }
-        public string CodexContractsId { get; }
-        public string GrafanaId { get; }
-    }
-}

Logging/BaseLog.cs

@@ -60,7 +60,6 @@ namespace Logging
         public virtual void AddStringReplace(string from, string to)
         {
             if (string.IsNullOrWhiteSpace(from)) return;
-            if (replacements.Any(r => r.From == from)) return;
             replacements.Add(new BaseLogStringReplacement(from, to));
         }
 
@@ -74,14 +73,6 @@ namespace Logging
             return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
         }
 
-        public void WriteLogTag()
-        {
-            var runId = NameUtils.GetRunId();
-            var category = NameUtils.GetCategoryName();
-            var name = NameUtils.GetTestMethodName();
-            LogFile.WriteRaw($"{runId} {category} {name}");
-        }
-
         private string ApplyReplacements(string str)
         {
             foreach (var replacement in replacements)
@@ -99,20 +90,20 @@ namespace Logging
 
     public class BaseLogStringReplacement
     {
+        private readonly string from;
+        private readonly string to;
+
         public BaseLogStringReplacement(string from, string to)
         {
-            From = from;
-            To = to;
+            this.from = from;
+            this.to = to;
 
             if (string.IsNullOrEmpty(from) || string.IsNullOrEmpty(to) || from == to) throw new ArgumentException();
         }
 
-        public string From { get; }
-        public string To { get; }
-
         public string Apply(string msg)
         {
-            return msg.Replace(From, To);
+            return msg.Replace(from, to);
         }
     }
 }

Logging/FixtureLog.cs

@@ -1,14 +1,20 @@
-namespace Logging
+using NUnit.Framework;
+
+namespace Logging
 {
     public class FixtureLog : BaseLog
     {
+        private readonly DateTime start;
         private readonly string fullName;
         private readonly LogConfig config;
 
-        public FixtureLog(LogConfig config, DateTime start, string name = "")
+        public FixtureLog(LogConfig config, string name = "")
             : base(config.DebugEnabled)
         {
-            fullName = NameUtils.GetFixtureFullName(config, start, name);
+            start = DateTime.UtcNow;
+            var folder = DetermineFolder(config);
+            var fixtureName = GetFixtureName(name);
+            fullName = Path.Combine(folder, fixtureName);
             this.config = config;
         }
 
@@ -26,5 +32,28 @@
         {
             return fullName;
         }
+
+        private string DetermineFolder(LogConfig config)
+        {
+            return Path.Join(
+               config.LogRoot,
+               $"{start.Year}-{Pad(start.Month)}",
+               Pad(start.Day));
+        }
+
+        private string GetFixtureName(string name)
+        {
+            var test = TestContext.CurrentContext.Test;
+            var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
+            if (!string.IsNullOrEmpty(name)) className = name;
+
+            return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
+        }
+
+        private static string Pad(int n)
+        {
+            return n.ToString().PadLeft(2, '0');
+        }
+
     }
 }

Logging/LogFile.cs

@@ -49,7 +49,7 @@
 
         private static string GetTimestamp()
         {
-            return $"[{DateTime.UtcNow.ToString("o")}]";
+            return $"[{DateTime.UtcNow.ToString("u")}]";
         }
 
         private void EnsurePathExists(string filename)

Logging/NameUtils.cs

@@ -1,85 +0,0 @@
-using NUnit.Framework;
-
-namespace Logging
-{
-    public static class NameUtils
-    {
-        public static string GetTestMethodName(string name = "")
-        {
-            if (!string.IsNullOrEmpty(name)) return name;
-            var test = TestContext.CurrentContext.Test;
-            var args = FormatArguments(test);
-            return ReplaceInvalidCharacters($"{test.MethodName}{args}");
-        }
-
-        public static string GetFixtureFullName(LogConfig config, DateTime start, string name)
-        {
-            var folder = DetermineFolder(config, start);
-            var fixtureName = GetFixtureName(name, start);
-            return Path.Combine(folder, fixtureName);
-        }
-
-        public static string GetRawFixtureName()
-        {
-            var test = TestContext.CurrentContext.Test;
-            if (test.ClassName!.Contains("AdhocContext")) return "none";
-            var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
-            return className.Replace('.', '-');
-        }
-
-        public static string GetCategoryName()
-        {
-            var test = TestContext.CurrentContext.Test;
-            if (test.ClassName!.Contains("AdhocContext")) return "none";
-            return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.'));
-        }
-
-        public static string GetTestId()
-        {
-            return GetEnvVar("TESTID");
-        }
-
-        public static string GetRunId()
-        {
-            return GetEnvVar("RUNID");
-        }
-
-        private static string GetEnvVar(string name)
-        {
-            var v = Environment.GetEnvironmentVariable(name);
-            if (string.IsNullOrEmpty(v)) return $"EnvVar-{name}-NotSet";
-            return v;
-        }
-
-        private static string FormatArguments(TestContext.TestAdapter test)
-        {
-            if (test.Arguments == null || !test.Arguments.Any()) return "";
-            return $"[{string.Join(',', test.Arguments)}]";
-        }
-
-        private static string ReplaceInvalidCharacters(string name)
-        {
-            return name.Replace(":", "_");
-        }
-
-        private static string DetermineFolder(LogConfig config, DateTime start)
-        {
-            return Path.Join(
-               config.LogRoot,
-               $"{start.Year}-{Pad(start.Month)}",
-               Pad(start.Day));
-        }
-
-        private static string GetFixtureName(string name, DateTime start)
-        {
-            var className = GetRawFixtureName();
-            if (!string.IsNullOrEmpty(name)) className = name;
-            return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
-        }
-
-        private static string Pad(int n)
-        {
-            return n.ToString().PadLeft(2, '0');
-        }
-    }
-}

Logging/NullLog.cs

@@ -6,11 +6,9 @@
         {
         }
 
-        public string FullFilename { get; set; } = "NULL";
-
         protected override string GetFullName()
         {
-            return FullFilename;
+            return "NULL";
         }
 
         public override void Log(string message)

Logging/StatusLog.cs

@@ -1,69 +0,0 @@
-using Newtonsoft.Json;
-
-namespace Logging
-{
-    public class StatusLog
-    {
-        private readonly object fileLock = new object();
-        private readonly string fullName;
-        private readonly string fixtureName;
-
-        public StatusLog(LogConfig config, DateTime start, string name = "")
-        {
-            fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log";
-            fixtureName = NameUtils.GetRawFixtureName();
-        }
-
-        public void ConcludeTest(string resultStatus, string testDuration, ApplicationIds applicationIds)
-        {
-            Write(new StatusLogJson
-            {
-                @timestamp = DateTime.UtcNow.ToString("o"),
-                runid = NameUtils.GetRunId(),
-                status = resultStatus,
-                testid = NameUtils.GetTestId(),
-                codexid = applicationIds.CodexId,
-                gethid = applicationIds.GethId,
-                prometheusid = applicationIds.PrometheusId,
-                codexcontractsid = applicationIds.CodexContractsId,
-                grafanaid = applicationIds.GrafanaId,
-                category = NameUtils.GetCategoryName(),
-                fixturename = fixtureName,
-                testname = NameUtils.GetTestMethodName(),
-                testduration = testDuration
-            });
-        }
-
-        private void Write(StatusLogJson json)
-        {
-            try
-            {
-                lock (fileLock)
-                {
-                    File.AppendAllLines(fullName, new[] { JsonConvert.SerializeObject(json) });
-                }
-            }
-            catch (Exception ex)
-            {
-                Console.WriteLine("Unable to write to status log: " + ex);
-            }
-        }
-    }
-
-    public class StatusLogJson
-    {
-        public string @timestamp { get; set; } = string.Empty;
-        public string runid { get; set; } = string.Empty;
-        public string status { get; set; } = string.Empty;
-        public string testid { get; set; } = string.Empty; 
-        public string codexid { get; set; } = string.Empty;
-        public string gethid { get; set; } = string.Empty;
-        public string prometheusid { get; set; } = string.Empty;
-        public string codexcontractsid { get; set; } = string.Empty;
-        public string grafanaid { get; set; } = string.Empty;
-        public string category { get; set; } = string.Empty;
-        public string fixturename { get; set; } = string.Empty;
-        public string testname { get; set; } = string.Empty;
-        public string testduration { get; set;} = string.Empty;
-    }
-}

Logging/TestLog.cs

@@ -10,7 +10,7 @@ namespace Logging
         public TestLog(string folder, bool debug, string name = "")
             : base(debug)
         {
-            methodName = NameUtils.GetTestMethodName(name);
+            methodName = GetMethodName(name);
             fullName = Path.Combine(folder, methodName);
 
             Log($"*** Begin: {methodName}");
@@ -37,5 +37,24 @@ namespace Logging
         {
             return fullName;
         }
+
+        private string GetMethodName(string name)
+        {
+            if (!string.IsNullOrEmpty(name)) return name;
+            var test = TestContext.CurrentContext.Test;
+            var args = FormatArguments(test);
+            return ReplaceInvalidCharacters($"{test.MethodName}{args}");
+        }
+
+        private static string FormatArguments(TestContext.TestAdapter test)
+        {
+            if (test.Arguments == null || !test.Arguments.Any()) return "";
+            return $"[{string.Join(',', test.Arguments)}]";
+        }
+
+        private static string ReplaceInvalidCharacters(string name)
+        {
+            return name.Replace(":", "_");
+        }
     }
 }

LongTests/BasicTests/DownloadTests.cs

@@ -1,5 +1,7 @@
 using DistTestCore;
 using NUnit.Framework;
+using k8s;
+using k8s.Models;
 
 namespace TestsLong.BasicTests
 {
@@ -23,7 +25,8 @@ namespace TestsLong.BasicTests
             var testFile = GenerateTestFile(filesizeMb.MB());
             var contentId = host.UploadFile(testFile);
             var list = new List<Task<TestFile?>>();
-
+            //sleep for 1 minute
+            Thread.Sleep(1200000);
             foreach (var node in group)
             {
                 list.Add(Task.Run(() => { return node.DownloadContent(contentId); }));

LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs

@@ -1,4 +1,5 @@
-using DistTestCore;
+using DistTestCore.Helpers;
+using DistTestCore;
 using NUnit.Framework;
 
 namespace TestsLong.DownloadConnectivityTests
@@ -15,7 +16,7 @@ namespace TestsLong.DownloadConnectivityTests
         {
             for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
 
-            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
+            PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
         }
     }
 }

README.md

@@ -3,6 +3,7 @@
 
 Using a common dotnet unit-test framework and a few other libraries, this project allows you to write tests that use multiple Codex node instances in various configurations to test the distributed system in a controlled, reproducible environment.
 
+
 Nim-Codex: https://github.com/codex-storage/nim-codex  
 Dotnet: v6.0  
 Kubernetes: v1.25.4  
@@ -22,7 +23,7 @@ Test executing can be configured using the following environment variables.
 |----------------|------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------|
 | KUBECONFIG     | Optional path (abs or rel) to kubeconfig YAML file. When null, uses system default (docker-desktop) kubeconfig if available.                         | (null)              |
 | LOGPATH        | Path (abs or rel) where log files will be saved.                                                                                                     | "CodexTestLogs"     |
-| LOGDEBUG       | When "true", enables additional test-runner debug log output.                                                                                        | "false"             |
+| LOGDEBUG       | When "true", enables additional test-runner debug log output.                                                                                                    | "false"             |
 | DATAFILEPATH   | Path (abs or rel) where temporary test data files will be saved.                                                                                     | "TestDataFiles"     |
 | LOGLEVEL       | Codex log-level. (case-insensitive)                                                                                                                  | "Trace"             |
 | RUNNERLOCATION | Use "ExternalToCluster" when test app is running outside of the k8s cluster. Use "InternalToCluster" when tests are run from inside a pod/container. | "ExternalToCluster" |
@@ -31,10 +32,10 @@ Test executing can be configured using the following environment variables.
 Because tests potentially take a long time to run, logging is in place to help you investigate failures afterwards. Should a test fail, all Codex terminal output (as well as metrics if they have been enabled) will be downloaded and stored along with a detailed, step-by-step log of the test. If something's gone wrong and you're here to discover the details, head for the logs.
 
 ## How to contribute tests
-An important goal of the test infra is to provide a simple, accessible way for developers to write their tests. If you want to contribute tests for Codex, please follow the steps [HERE](/CONTRIBUTINGTESTS.md).
+An important goal of the test infra is to provide a simple, accessible way for developers to write their tests. If you want to contribute tests for Codex, please follow the steps [HERE](/CONTRIBUTINGTESTS.MD).
 
 ## Run the tests on your machine
-Creating tests is much easier when you can debug them on your local system. This is possible, but requires some set-up. If you want to be able to run the tests on your local system, follow the steps [HERE](/docs/LOCALSETUP.md). Please note that tests which require explicit node locations cannot be executed locally. (Well, you could comment out the location statements and then it would probably work. But that might impact the validity/usefulness of the test.)
+Creating tests is much easier when you can debug them on your local system. This is possible, but requires some set-up. If you want to be able to run the tests on your local system, follow the steps [HERE](/LOCALSETUP.MD). Please note that tests which require explicit node locations cannot be executed locally. (Well, you could comment out the location statements and then it would probably work. But that might impact the validity/usefulness of the test.)
 
 ## Missing functionality
 Surely the test-infra doesn't do everything we'll need it to do. If you're running into a limitation and would like to request a new feature for the test-infra, please create an issue.

Tests/BasicTests/ContinuousSubstitute.cs

@@ -1,252 +0,0 @@
-using DistTestCore;
-using NUnit.Framework;
-using Utils;
-
-namespace Tests.BasicTests
-{
-    [Ignore("Used for debugging continuous tests")]
-    [TestFixture]
-    public class ContinuousSubstitute : AutoBootstrapDistTest
-    {
-        [Test]
-        public void ContinuousTestSubstitute()
-        {
-            var group = SetupCodexNodes(5, o => o
-                    .EnableMetrics()
-                    .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
-                    .WithBlockTTL(TimeSpan.FromMinutes(2))
-                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
-                    .WithBlockMaintenanceNumber(10000)
-                    .WithBlockTTL(TimeSpan.FromMinutes(2))
-                    .WithStorageQuota(1.GB()));
-
-            var nodes = group.Cast<OnlineCodexNode>().ToArray();
-
-            foreach (var node in nodes)
-            {
-                node.Marketplace.MakeStorageAvailable(
-                size: 500.MB(),
-                minPricePerBytePerSecond: 1.TestTokens(),
-                maxCollateral: 1024.TestTokens(),
-                maxDuration: TimeSpan.FromMinutes(5));
-            }
-
-            var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
-            while (DateTime.UtcNow < endTime)
-            {
-                var allNodes = nodes.ToList();
-                var primary = allNodes.PickOneRandom();
-                var secondary = allNodes.PickOneRandom();
-
-                Log("Run Test");
-                PerformTest(primary, secondary);
-
-                Thread.Sleep(TimeSpan.FromSeconds(5));
-            }
-        }
-
-        [Test]
-        public void PeerTest()
-        {
-            var group = SetupCodexNodes(5, o => o
-                    .EnableMetrics()
-                    .EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true)
-                    .WithBlockTTL(TimeSpan.FromMinutes(2))
-                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
-                    .WithBlockMaintenanceNumber(10000)
-                    .WithBlockTTL(TimeSpan.FromMinutes(2))
-                    .WithStorageQuota(1.GB()));
-
-            var nodes = group.Cast<OnlineCodexNode>().ToArray();
-
-            var checkTime = DateTime.UtcNow + TimeSpan.FromMinutes(1);
-            var endTime = DateTime.UtcNow + TimeSpan.FromHours(10);
-            while (DateTime.UtcNow < endTime)
-            {
-                CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
-                CheckRoutingTables(GetAllOnlineCodexNodes());
-
-                var node = RandomUtils.PickOneRandom(nodes.ToList());
-                var file = GenerateTestFile(50.MB());
-                node.UploadFile(file);
-
-                Thread.Sleep(20000);
-            }
-        }
-
-        private void CheckRoutingTables(IEnumerable<IOnlineCodexNode> nodes)
-        {
-            var all = nodes.ToArray();
-            var allIds = all.Select(n => n.GetDebugInfo().table.localNode.nodeId).ToArray();
-
-            var errors = all.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray();
-            
-            if (errors.Any())
-            {
-                Assert.Fail(string.Join(Environment.NewLine, errors));
-            }
-        }
-
-        private string AreAllPresent(IOnlineCodexNode n, string[] allIds)
-        {
-            var info = n.GetDebugInfo();
-            var known = info.table.nodes.Select(n => n.nodeId).ToArray();
-            var expected = allIds.Where(i => i != info.table.localNode.nodeId).ToArray();
-
-            if (!expected.All(ex => known.Contains(ex)))
-            {
-                return $"Not all of '{string.Join(",", expected)}' were present in routing table: '{string.Join(",", known)}'";
-            }
-
-            return string.Empty;
-        }
-
-        private ByteSize fileSize = 80.MB();
-
-        private void PerformTest(IOnlineCodexNode primary, IOnlineCodexNode secondary)
-        {
-            ScopedTestFiles(() =>
-            {
-                var testFile = GenerateTestFile(fileSize);
-
-                var contentId = primary.UploadFile(testFile);
-
-                var downloadedFile = secondary.DownloadContent(contentId);
-
-                testFile.AssertIsEqual(downloadedFile);
-            });
-        }
-        
-        [Test]
-        public void HoldMyBeerTest()
-        {
-            var blockExpirationTime = TimeSpan.FromMinutes(3);
-            var group = SetupCodexNodes(3, o => o
-                    .EnableMetrics()
-                    .WithBlockTTL(blockExpirationTime)
-                    .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2))
-                    .WithBlockMaintenanceNumber(10000)
-                    .WithStorageQuota(2000.MB()));
-
-            var nodes = group.Cast<OnlineCodexNode>().ToArray();
-
-            var endTime = DateTime.UtcNow + TimeSpan.FromHours(24);
-
-            var filesize = 80.MB();
-            double codexDefaultBlockSize = 31 * 64 * 33;
-            var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize));
-            var sizeInBytes = filesize.SizeInBytes;
-            Assert.That(numberOfBlocks, Is.EqualTo(1282));
-
-            var startTime = DateTime.UtcNow;
-            var successfulUploads = 0;
-            var successfulDownloads = 0;
-
-            while (DateTime.UtcNow < endTime)
-            {
-                foreach (var node in nodes)
-                {
-                    try
-                    {
-                        Thread.Sleep(TimeSpan.FromSeconds(5));
-
-                        ScopedTestFiles(() =>
-                        {
-                            var uploadStartTime = DateTime.UtcNow;
-                            var file = GenerateTestFile(filesize);
-                            var cid = node.UploadFile(file);
-
-                            var cidTag = cid.Id.Substring(cid.Id.Length - 6);
-                            Measure("upload-log-asserts", () =>
-                            {
-                                var uploadLog = node.DownloadLog(tailLines: 50000);
-
-                                var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\"");
-                                uploadLog.DeleteFile();
-
-                                var storeLine = GetLineForCidTag(storeLines, cidTag);
-                                AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes);
-                            });
-                            successfulUploads++;
-
-                            var uploadTimeTaken = DateTime.UtcNow - uploadStartTime;
-                            if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10)))
-                            {
-                                Assert.Fail("Upload took too long. Blocks already expired.");
-                            }
-
-                            var dl = node.DownloadContent(cid);
-                            file.AssertIsEqual(dl);
-
-                            Measure("download-log-asserts", () =>
-                            {
-                                var downloadLog = node.DownloadLog(tailLines: 50000);
-
-                                var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\"");
-                                downloadLog.DeleteFile();
-
-                                var sentLine = GetLineForCidTag(sentLines, cidTag);
-                                AssertSentLineContains(sentLine, sizeInBytes);
-                            });
-                            successfulDownloads++;
-                        });
-                    }
-                    catch
-                    {
-                        var testDuration = DateTime.UtcNow - startTime;
-                        Log("Test failed. Delaying shut-down by 30 seconds to collect metrics.");
-                        Log($"Test failed after {Time.FormatDuration(testDuration)} and {successfulUploads} successful uploads and {successfulDownloads} successful downloads");
-                        Thread.Sleep(TimeSpan.FromSeconds(30));
-                        throw;
-                    }
-                }
-
-                Thread.Sleep(TimeSpan.FromSeconds(5));
-            }
-        }
-
-        private void AssertSentLineContains(string sentLine, long sizeInBytes)
-        {
-            var tag = "bytes=";
-            var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length);
-            var bytes = Convert.ToInt64(token);
-            Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'");
-        }
-
-        private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes)
-        {
-            var tokens = storeLine.Split(" ");
-
-            var blocksToken = GetToken(tokens, "blocks=");
-            var sizeToken = GetToken(tokens, "size=");
-            if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine);
-            if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine);
-
-            var blocks = Convert.ToInt64(blocksToken);
-            var size = Convert.ToInt64(sizeToken?.Replace("'NByte", ""));
-
-            var lineLog = $" Line: '{storeLine}'";
-            Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog);
-            Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog);
-        }
-
-        private string GetLineForCidTag(string[] lines, string cidTag)
-        {
-            var result = lines.SingleOrDefault(l => l.Contains(cidTag));
-            if (result == null)
-            {
-                Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'");
-                throw new Exception();
-            }
-
-            return result;
-        }
-
-        private string? GetToken(string[] tokens, string tag)
-        {
-            var token = tokens.SingleOrDefault(t => t.StartsWith(tag));
-            if (token == null) return null;
-            return token.Substring(tag.Length);
-        }
-    }
-}

Tests/BasicTests/ExampleTests.cs

@@ -1,5 +1,6 @@
 using DistTestCore;
 using NUnit.Framework;
+using Utils;
 
 namespace Tests.BasicTests
 {
@@ -43,7 +44,6 @@ namespace Tests.BasicTests
         {
             var sellerInitialBalance = 234.TestTokens();
             var buyerInitialBalance = 1000.TestTokens();
-            var fileSize = 10.MB();
 
             var seller = SetupCodexNode(s => s
                             .WithStorageQuota(11.GB())
@@ -56,27 +56,25 @@ namespace Tests.BasicTests
                 maxCollateral: 20.TestTokens(),
                 maxDuration: TimeSpan.FromMinutes(3));
 
-            var testFile = GenerateTestFile(fileSize);
+            var testFile = GenerateTestFile(10.MB());
 
             var buyer = SetupCodexNode(s => s
                 .WithBootstrapNode(seller)
                 .EnableMarketplace(buyerInitialBalance));
-
-            buyer.Marketplace.AssertThatBalance(Is.EqualTo(buyerInitialBalance));
             
             var contentId = buyer.UploadFile(testFile);
-            var purchaseContract = buyer.Marketplace.RequestStorage(contentId,
+            buyer.Marketplace.RequestStorage(contentId,
                 pricePerSlotPerSecond: 2.TestTokens(),
                 requiredCollateral: 10.TestTokens(),
                 minRequiredNumberOfNodes: 1,
                 proofProbability: 5,
                 duration: TimeSpan.FromMinutes(1));
 
-            purchaseContract.WaitForStorageContractStarted(fileSize);
+            Time.Sleep(TimeSpan.FromSeconds(10));
 
             seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed.");
 
-            purchaseContract.WaitForStorageContractFinished();
+            Time.Sleep(TimeSpan.FromMinutes(1));
 
             seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage.");
             buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage.");

Tests/BasicTests/MixedTests.cs

@@ -0,0 +1,49 @@
+using DistTestCore;
+using NUnit.Framework;
+
+namespace Tests.ParallelTests
+{
+    [TestFixture]
+    public class MixedTests : DistTest
+    {
+        [TestCase(1, 10)]
+        [UseLongTimeouts]
+        public void ParallelMixed(int numberOfNodes, int filesizeMb)
+        {
+            // initialize the nodes
+            var group = SetupCodexNodes(numberOfNodes);
+            var host = SetupCodexNode();
+
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+            // Upload single file for the download nodes
+            var testfile = GenerateTestFile(filesizeMb.MB());
+            var contentId = host.UploadFile(testfile);
+
+            var testfiles = new List<TestFile>();
+            var contentIds = new List<Task<ContentId>>();
+
+            // Starts uploads for the upload nodes
+            for (int i = 0; i < group.Count(); i++)
+            {
+                testfiles.Add(GenerateTestFile(filesizeMb.MB()));
+                var n = i;
+                contentIds.Add(Task.Run(() => { return host.UploadFile(testfiles[n]); }));
+            }
+            // Starts downloads for the download nodes
+            var downloads = new List<Task<TestFile?>>();
+            for (int i = 0; i < group.Count(); i++)
+            {
+                var n = i;
+                downloads.Add(Task.Run(() => { return group[n].DownloadContent(contentId); }));
+            }
+            Task.WaitAll(downloads.ToArray());
+            for (int i = 0; i < group.Count(); i++)
+            {
+                testfiles[i].AssertIsEqual(downloads[i].Result);
+            }
+        }
+    }
+}

Tests/BasicTests/TwoClientTests.cs

@@ -8,7 +8,7 @@ namespace Tests.BasicTests
     public class TwoClientTests : DistTest
     {
         [Test]
-        public void TwoClientTest()
+        public void TwoClientsOnePodTest()
         {
             var group = SetupCodexNodes(2);
 
@@ -18,6 +18,15 @@ namespace Tests.BasicTests
             PerformTwoClientTest(primary, secondary);
         }
 
+        [Test]
+        public void TwoClientsTwoPodsTest()
+        {
+            var primary = SetupCodexNode();
+            var secondary = SetupCodexNode();
+
+            PerformTwoClientTest(primary, secondary);
+        }
+
         [Test]
         public void TwoClientsTwoLocationsTest()
         {

Tests/DownloadConnectivityTests/FulllyConnectedFetchTests.cs

@@ -1,38 +0,0 @@
-using DistTestCore;
-using NUnit.Framework;
-
-namespace Tests.DownloadConnectivityTests
-{
-    public class FulllyConnectedFetchTests : AutoBootstrapDistTest
-    {
-        [Test]
-        public void MetricsDoesNotInterfereWithFetch()
-        {
-            SetupCodexNodes(2, s => s.EnableMetrics());
-
-            AssertAllNodesConnected();
-        }
-
-        [Test]
-        public void MarketplaceDoesNotInterfereWithFetch()
-        {
-            SetupCodexNodes(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens()));
-
-            AssertAllNodesConnected();
-        }
-
-        [Test]
-        [Combinatorial]
-        public void FullyConnectedFetchTest([Values(1, 3, 5)] int numberOfNodes)
-        {
-            SetupCodexNodes(numberOfNodes);
-
-            AssertAllNodesConnected();
-        }
-
-        private void AssertAllNodesConnected()
-        {
-            CreatePeerFetchTestHelpers().AssertFullFetchInterconnectivity(GetAllOnlineCodexNodes());
-        }
-    }
-}

Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -6,36 +6,15 @@ namespace Tests.DownloadConnectivityTests
     [TestFixture]
     public class FullyConnectedDownloadTests : AutoBootstrapDistTest
     {
-        [Test]
-        public void MetricsDoesNotInterfereWithPeerDownload()
-        {
-            SetupCodexNodes(2, s => s.EnableMetrics());
-
-            AssertAllNodesConnected();
-        }
-
-        [Test]
-        public void MarketplaceDoesNotInterfereWithPeerDownload()
-        {
-            SetupCodexNodes(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens()));
-
-            AssertAllNodesConnected();
-        }
-
         [Test]
         [Combinatorial]
         public void FullyConnectedDownloadTest(
             [Values(1, 3, 5)] int numberOfNodes,
             [Values(1, 10)] int sizeMBs)
         {
-            SetupCodexNodes(numberOfNodes);
+            for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
 
-            AssertAllNodesConnected(sizeMBs);
-        }
-
-        private void AssertAllNodesConnected(int sizeMBs = 10)
-        {
-            CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
+            PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
         }
     }
 }

Tests/MembershipChangeTest/BasicTests.cs

@@ -0,0 +1,130 @@
+using DistTestCore;
+using KubernetesWorkflow;
+using NUnit.Framework;
+namespace Tests.MembershipChangeTests
+{
+    [TestFixture]
+    public class MembershipChangeTests : DistTest
+    {
+
+        [Test]
+        public void SingleDownloadWhileAdding()
+        {
+            var filesize = 100.MB();
+
+            var group = SetupCodexNodes(1);
+            var toAdd = SetupCodexNodes(5);
+            var host = SetupCodexNodes(1)[0];
+
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+
+            var testFile = GenerateTestFile(filesize);
+            var contentId = host.UploadFile(testFile);
+            var list = new List<Task>();
+            foreach (var node in toAdd)
+            {
+                list.Add(Task.Run(() => { host.ConnectToPeer(node); }));
+            }
+
+            var resFile = group[0].DownloadContent(contentId);
+            Task.WaitAll(list.ToArray());
+            testFile.AssertIsEqual(resFile);
+        }
+        [Test]
+        public void SingleUploadWhileAdding()
+        {
+            var filesize = 100.MB();
+
+            var group = SetupCodexNodes(1);
+            var toAdd = SetupCodexNodes(5);
+            var host = SetupCodexNodes(1)[0];
+
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+
+            var testFile = GenerateTestFile(filesize);
+            var list = new List<Task>();
+            foreach (var node in toAdd)
+            {
+                list.Add(Task.Run(() => { host.ConnectToPeer(node); }));
+            }
+            var contentId = host.UploadFile(testFile);
+
+
+            Task.WaitAll(list.ToArray());
+            var resFile = group[0].DownloadContent(contentId);
+            testFile.AssertIsEqual(resFile);
+        }
+        [Test]
+        public void SingleDownloadMixedMembership()
+        {
+            var filesize = 100.MB();
+
+            var group = SetupCodexNodes(1);
+            var toAdd = SetupCodexNodes(5);
+            var toRemove = SetupCodexNodes(5);
+            var host = SetupCodexNodes(1)[0];
+
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+            foreach (var node in toRemove)
+            {
+                host.ConnectToPeer(node);
+            }
+
+
+            var testFile = GenerateTestFile(filesize);
+            var contentId = host.UploadFile(testFile);
+
+            for (var i = 0; i < toAdd.Count(); i++)
+            {
+                Task.Run(() => { host.ConnectToPeer(toAdd[i]); });
+                Task.Run(() => { toRemove[i].BringOffline(); });
+            }
+
+            var resFile = group[0].DownloadContent(contentId);
+            testFile.AssertIsEqual(resFile);
+        }
+        [Test]
+        public void SingleUploadMixedMembership()
+        {
+            var filesize = 100.MB();
+
+            var group = SetupCodexNodes(1);
+            var toAdd = SetupCodexNodes(5);
+            var toRemove = SetupCodexNodes(5);
+            var host = SetupCodexNodes(1)[0];
+
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+            foreach (var node in toRemove)
+            {
+                host.ConnectToPeer(node);
+            }
+
+            var testFile = GenerateTestFile(filesize);
+
+            var list = new List<Task>();
+
+            for (var i = 0; i < toAdd.Count(); i++)
+            {
+                Task.Run(() => { host.ConnectToPeer(toAdd[i]); });
+                Task.Run(() => { toRemove[i].BringOffline(); });
+            }
+            var contentId = host.UploadFile(testFile);
+
+            var resFile = group[0].DownloadContent(contentId);
+            testFile.AssertIsEqual(resFile);
+        }
+    }
+
+}

Tests/MembershipChangeTest/DownloadTests.cs

@@ -0,0 +1,67 @@
+using DistTestCore;
+using KubernetesWorkflow;
+using NUnit.Framework;
+namespace Tests.MembershipChangeTests
+{
+    [TestFixture]
+    public class DownloadMembershipChangeTests : DistTest
+    {
+        [TestCase(1, 100, 5, 0)]
+        [TestCase(1, 100, 0, 5)]
+        [TestCase(1, 100, 5, 5)]
+        [UseLongTimeouts]
+        public void DownloadMembershipChange(int numberOfNodes, int filesize, int numberOfNodesToAdd = 0, int numberOfNodesToRemove = 0)
+        {
+            // Setup 3 node groups, one which will be added during the procedure, one which will be dropped during the procedure, and the one being tested. 
+            ICodexNodeGroup? toAdd = null;
+            ICodexNodeGroup? toRemove = null;
+            var group = SetupCodexNodes(numberOfNodes);
+            if (numberOfNodesToAdd != 0)
+                toAdd = SetupCodexNodes(numberOfNodesToAdd);
+            if (numberOfNodesToRemove != 0)
+                toRemove = SetupCodexNodes(numberOfNodesToRemove);
+            var host = SetupCodexNodes(1)[0];
+
+            // Connect the main and dropping nodes to the host
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+            if (toRemove != null)
+                foreach (var node in toRemove)
+                    host.ConnectToPeer(node);
+
+            // Upload a file to the host
+            var testFile = GenerateTestFile(filesize.MB());
+            var contentId = host.UploadFile(testFile);
+
+            var list = new List<Task<TestFile?>>();
+                        
+            // Start the download for each node
+            foreach (var node in group)
+            {
+                list.Add(Task.Run(() => { return node.DownloadContent(contentId); }));
+            }
+
+            // Start adding and dropping nodes during the download
+            // TODO: The log is put here for debug, but without it the members do not run async
+            for (var i = 0; (toAdd != null && i < toAdd.Count()) || (toRemove != null && i < toRemove.Count()); i++)
+            {
+                Log($"Iteration {i}");
+                if (toAdd != null && i < toAdd.Count())
+                    Task.Run(() => { host.ConnectToPeer(toAdd[i]); });
+                if (toRemove != null && i < toRemove.Count())
+                    Task.Run(() => { toRemove[i].BringOffline(); });
+            }
+
+            // Wait for the download to finish
+            Task.WaitAll(list.ToArray());
+
+            // Assert that the download was successful
+            foreach (var task in list)
+            {
+                testFile.AssertIsEqual(task.Result);
+            }
+        }
+    }
+}

Tests/MembershipChangeTest/MixedTests.cs

@@ -0,0 +1,83 @@
+using DistTestCore;
+using KubernetesWorkflow;
+using NUnit.Framework;
+namespace Tests.MembershipChangeTests
+{
+    [TestFixture]
+    public class MixedMembershipChangeTests : DistTest
+    {
+        [TestCase(1, 5, 0, 0)]
+
+        [UseLongTimeouts]
+        public void MixedMembershipChange(int numberOfNodes, int filesize, int numberOfNodesToAdd, int numberOfNodesToRemove)
+        {
+            // Creating the node groups
+            ICodexNodeGroup? toAdd = null;
+            ICodexNodeGroup? toAddSecondary = null;
+            ICodexNodeGroup? toRemove = null;
+            var group = SetupCodexNodes(numberOfNodes);
+            
+            if (numberOfNodesToAdd != 0) {
+                toAdd = SetupCodexNodes(numberOfNodesToAdd);
+                toAddSecondary = SetupCodexNodes(numberOfNodesToAdd);
+            }
+            if (numberOfNodesToRemove != 0)
+                toRemove = SetupCodexNodes(numberOfNodesToRemove);
+            var host = SetupCodexNodes(1)[0];
+
+            // Connect the main and dropping nodes to the host
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+            if (toRemove != null)
+                foreach (var node in toRemove)
+                    host.ConnectToPeer(node);
+
+            var testFile = GenerateTestFile(filesize.MB());
+            var contentId = host.UploadFile(testFile);
+
+            var testfiles = new List<TestFile>();
+            var contentIds = new List<Task<ContentId>>();
+
+            var downloads = new List<Task<TestFile?>>();
+
+            // Start adding and dropping nodes
+            for (var i = 0; (toAdd != null && i < toAdd.Count()) || (toRemove != null && i < toRemove.Count()); i++)
+            {
+                Log($"Iteration {i}");
+                if (toAdd != null && i < toAdd.Count())
+                    Task.Run(() => { host.ConnectToPeer(toAdd[i]); });
+                if (toRemove != null && i < toRemove.Count())
+                    Task.Run(() => { toRemove[i].BringOffline(); });
+            }
+
+            // Start the upload for each node in the main group
+            for (int i = 0; i < group.Count(); i++)
+            {
+                testfiles.Add(GenerateTestFile(filesize.MB()));
+                var n = i;
+                contentIds.Add(Task.Run(() => { return host.UploadFile(testfiles[n]); }));
+            }
+
+            // Start the download for each node in the main group
+            for (int i = 0; i < group.Count(); i++)
+            {
+                var n = i;
+                downloads.Add(Task.Run(() => { return group[n].DownloadContent(contentId); }));
+            }
+
+            // Wait for the download to finish
+            Task.WaitAll(downloads.ToArray());
+
+            // Wait for the upload to finish
+            Task.WaitAll(contentIds.ToArray());
+
+            // Assert that the files are intact
+            for (int i = 0; i < group.Count(); i++)
+            {
+                testfiles[i].AssertIsEqual(downloads[i].Result);
+            }
+        }
+    }
+}

Tests/MembershipChangeTest/UploadTests.cs

@@ -0,0 +1,78 @@
+using DistTestCore;
+using KubernetesWorkflow;
+using NUnit.Framework;
+namespace Tests.MembershipChangeTests
+{
+     [TestFixture]
+    public class UploadMembershipChangeTests : DistTest
+    {
+        [TestCase(1, 100, 5, 0)]
+        [TestCase(1, 100, 0, 5)]
+        [TestCase(1, 100, 5, 5)]
+
+        [UseLongTimeouts]
+        public void UploadMembershipChange(int numberOfNodes, int filesize, int numberOfNodesToAdd = 0, int numberOfNodesToRemove = 0)
+        {
+            // Creating the node groups
+            ICodexNodeGroup? toAdd = null;
+            ICodexNodeGroup? toRemove = null;
+            var group = SetupCodexNodes(numberOfNodes);
+            if (numberOfNodesToAdd != 0)
+                toAdd = SetupCodexNodes(numberOfNodesToAdd);
+            if (numberOfNodesToRemove != 0)
+                toRemove = SetupCodexNodes(numberOfNodesToRemove);
+            var host = SetupCodexNodes(1)[0];
+
+            // Connect the main and dropping nodes to the host
+            foreach (var node in group)
+            {
+                host.ConnectToPeer(node);
+            }
+            if (toRemove != null)
+                foreach (var node in toRemove)
+                    host.ConnectToPeer(node);
+
+
+            var testfiles = new List<TestFile>();
+            var contentIds = new List<Task<ContentId>>();
+
+            // Start adding and dropping nodes
+
+            // Start the upload for each node in the main group
+            for (int i = 0; i < group.Count(); i++)
+            {
+                testfiles.Add(GenerateTestFile(filesize.MB()));
+                var n = i;
+                contentIds.Add(Task.Run(() => { return host.UploadFile(testfiles[n]); }));
+            }
+            for (var i = 0; (toAdd != null && i < toAdd.Count()) || (toRemove != null && i < toRemove.Count()); i++)
+            {
+                Log($"Iteration {i}");
+                if (toAdd != null && i < toAdd.Count())
+                    Task.Run(() => { host.ConnectToPeer(toAdd[i]); });
+                if (toRemove != null && i < toRemove.Count())
+                    Task.Run(() => { toRemove[i].BringOffline(); });
+            }
+
+            // Wait for the upload to finish
+            Task.WaitAll(contentIds.ToArray());
+
+            // Download the files
+            var downloads = new List<Task<TestFile?>>();
+            for (int i = 0; i < group.Count(); i++)
+            {
+                var n = i;
+                downloads.Add(Task.Run(() => { return group[n].DownloadContent(contentIds[n].Result); }));
+            }
+
+            // Wait for the download to finish
+            Task.WaitAll(downloads.ToArray());
+
+            // Assert that the files are intact
+            for (int i = 0; i < group.Count(); i++)
+            {
+                testfiles[i].AssertIsEqual(downloads[i].Result);
+            }
+        }
+    }
+}