Merge branch 'master' into feature/public-testnet-deploying

.github/workflows/continuous-tests.yaml

@@ -14,16 +14,16 @@ on:
   #     - '!docker/continuous-tests-job.yaml'
   workflow_dispatch:
     inputs:
-      branch:
-        description: Branch (master)
-        required: false
-        type: string
       source:
         description: Repository with tests (current)
         required: false
         type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
       nameprefix:
-        description: Runner name prefix (codex-continuous-tests)
+        description: Runner name prefix (c-tests-runner)
         required: false
         type: string
       namespace:
@@ -31,7 +31,7 @@ on:
         required: false
         type: string
       tests_target_duration:
-        description: Runner target duration (172800s=48h)
+        description: Runner target duration (172800 = 48h)
         required: false
         type: string
       tests_filter:
@@ -39,11 +39,14 @@ on:
         required: false
         type: string
       tests_cleanup:
-        description: Runner tests cleanup (true)
+        description: Runner tests cleanup
-        required: false
+        type: choice
-        type: string
+        options:
+        - true
+        - false
+        default: true
       deployment_namespace:
-        description: Deployment namespace (codex-continuous-tests)
+        description: Deployment namespace (c-tests-$runid)
         required: false
         type: string
 
@@ -51,9 +54,9 @@ on:
 env:
   BRANCH: ${{ github.ref_name }}
   SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
-  NAMEPREFIX: codex-continuous-tests
+  NAMEPREFIX: c-tests-runner
   NAMESPACE: default
-  DEPLOYMENT_NAMESPACE: codex-continuous-tests
+  DEPLOYMENT_NAMESPACE: c-tests
   TESTS_TARGET_DURATION: 172800
   TESTS_FILTER: ""
   TESTS_CLEANUP: true
@@ -72,16 +75,18 @@ jobs:
 
       - name: Variables
         run: |
-          [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
+          RUNID=$(date +%Y%m%d-%H%M%S)
+          echo "RUNID=${RUNID}" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
           [[ -n "${{ github.event.inputs.source }}" ]] && echo "SOURCE=${{ github.event.inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.branch }}" ]] && echo "BRANCH=${{ github.event.inputs.branch }}" >>"$GITHUB_ENV" || echo "BRANCH=${{ env.BRANCH }}" >>"$GITHUB_ENV"
           [[ -n "${{ github.event.inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ github.event.inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
           [[ -n "${{ github.event.inputs.namespace }}" ]] && echo "NAMESPACE=${{ github.event.inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
           [[ -n "${{ github.event.inputs.tests_target_duration }}" ]] && echo "TESTS_TARGET_DURATION=${{ github.event.inputs.tests_target_duration }}" >>"$GITHUB_ENV" || echo "TESTS_TARGET_DURATION=${{ env.TESTS_TARGET_DURATION }}" >>"$GITHUB_ENV"
           [[ -n "${{ github.event.inputs.tests_filter }}" ]] && echo "TESTS_FILTER=${{ github.event.inputs.tests_filter }}" >>"$GITHUB_ENV" || echo "TESTS_FILTERS=${{ env.TESTS_FILTERS }}" >>"$GITHUB_ENV"
           [[ -n "${{ github.event.inputs.tests_cleanup }}" ]] && echo "TESTS_CLEANUP=${{ github.event.inputs.tests_cleanup }}" >>"$GITHUB_ENV" || echo "TESTS_CLEANUP=${{ env.TESTS_CLEANUP }}" >>"$GITHUB_ENV"
-          [[ -n "${{ github.event.inputs.deployment_namespace }}" ]] && echo "DEPLOYMENT_NAMESPACE=${{ github.event.inputs.deployment_namespace }}" >>"$GITHUB_ENV" || echo "DEPLOYMENT_NAMESPACE=${{ env.DEPLOYMENT_NAMESPACE }}" >>"$GITHUB_ENV"
+          [[ -n "${{ github.event.inputs.deployment_namespace }}" ]] && echo "DEPLOYMENT_NAMESPACE=${{ github.event.inputs.deployment_namespace }}" >>"$GITHUB_ENV" || echo "DEPLOYMENT_NAMESPACE=${{ env.DEPLOYMENT_NAMESPACE }}-${RUNID}" >>"$GITHUB_ENV"
-          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
+
-          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
       - name: Kubectl - Install ${{ env.KUBE_VERSION }}
         uses: azure/setup-kubectl@v3
         with:
@@ -91,6 +96,22 @@ jobs:
         run: |
           mkdir -p "${HOME}"/.kube
           echo "${{ env.KUBE_CONFIG }}" | base64 -d > "${HOME}"/.kube/config
+
       - name: Kubectl - Create Job
         run: |
           envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner Pod: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+          echo "Tests runid: ${{ env.RUNID }}"
+          echo "Tests namespace: ${{ env.DEPLOYMENT_NAMESPACE }}"
+          echo "Tests duration: ${{ env.TESTS_TARGET_DURATION }}"
+          echo "Tests filter: ${{ env.TESTS_FILTER }}"
+          echo "Tests cleanup: ${{ env.TESTS_CLEANUP }}"
+          echo "----"

.github/workflows/dist-tests.yaml

@@ -14,16 +14,16 @@ on:
   #     - '!docker/dist-tests-job.yaml'
   workflow_dispatch:
     inputs:
-      branch:
-        description: Branch (master)
-        required: false
-        type: string
       source:
         description: Repository with tests (current)
         required: false
         type: string
+      branch:
+        description: Branch with tests (master)
+        required: false
+        type: string
       nameprefix:
-        description: Runner prefix (codex-dist-tests)
+        description: Runner name prefix (d-tests-runner)
         required: false
         type: string
       namespace:
@@ -31,7 +31,7 @@ on:
         required: false
         type: string
       command:
-        description: Runner command (dotnet test Tests)
+        description: Command (dotnet test Tests/CodexTests)
         required: false
         type: string
 
@@ -39,7 +39,7 @@ on:
 env:
   BRANCH: ${{ github.ref_name }}
   SOURCE: ${{ format('{0}/{1}', github.server_url, github.repository) }}
-  NAMEPREFIX: codex-dist-tests
+  NAMEPREFIX: d-tests-runner
   NAMESPACE: default
   COMMAND: dotnet test Tests/CodexTests
   JOB_MANIFEST: docker/dist-tests-job.yaml
@@ -79,3 +79,15 @@ jobs:
       - name: Kubectl - Create Job
         run: |
           envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -
+
+      - name: Tests Identification
+        run: |
+          echo "----"
+          echo "Repository: ${{ env.SOURCE }}"
+          echo "Branch: ${{ env.BRANCH }}"
+          echo "Runner Pod: ${{ env.NAMEPREFIX }}-${{ env.RUNID }}"
+          echo "Runner namespace: ${{ env.NAMESPACE }}"
+          echo "----"
+          echo "Tests runid: ${{ env.RUNID }}"
+          echo "Tests command: `jq -r '. | join(" ")' <<< '${{ env.COMMAND }}'`"
+          echo "----"

Framework/Core/CoreInterface.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace Core
 {

Framework/KubernetesWorkflow/K8sCluster.cs

@@ -54,16 +54,4 @@ namespace KubernetesWorkflow
             }
         }
     }
-
-    public class K8sNodeLabel
-    {
-        public K8sNodeLabel(string key, string value)
-        {
-            Key = key;
-            Value = value;
-        }
-
-        public string Key { get; }
-        public string Value { get; }
-    }
 }

Framework/KubernetesWorkflow/K8sController.cs

@@ -1,5 +1,7 @@
 using k8s;
 using k8s.Models;
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
 using Logging;
 using Utils;
 
@@ -623,11 +625,28 @@ namespace KubernetesWorkflow
         }
 
         private V1Pod GetPodForDeployment(RunningDeployment deployment)
+        {
+            return Time.Retry(() => GetPodForDeplomentInternal(deployment),
+                maxRetries: 2,
+                retryTime: TimeSpan.FromSeconds(10),
+                description: "Find pod by label for deployment.");
+        }
+
+        private V1Pod GetPodForDeplomentInternal(RunningDeployment deployment)
         {
             var allPods = client.Run(c => c.ListNamespacedPod(K8sNamespace));
             var pods = allPods.Items.Where(p => p.GetLabel(PodLabelKey) == deployment.PodLabel).ToArray();
 
-            if (pods.Length != 1) throw new Exception("Expected to find only 1 pod by podLabel.");
+            if (pods.Length != 1)
+            {
+                var allLabels = allPods.Items.Select(p =>
+                {
+                    var labels = string.Join(",", p.Labels().Select(l => $"{l.Key}={l.Value}"));
+                    return $"pod:'{p.Name()}' has labels: [{labels}]";
+                });
+                throw new Exception($"Expected to find 1 pod by podLabel '{deployment.PodLabel}'. Found: {pods.Length}. " +
+                    $"Total number of pods: {allPods.Items.Count}. Their labels: {string.Join(Environment.NewLine, allLabels)}");
+            }
             return pods[0];
         }
 

Framework/KubernetesWorkflow/K8sHooks.cs

@@ -1,4 +1,7 @@
-namespace KubernetesWorkflow
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
+
+namespace KubernetesWorkflow
 {
     public interface IK8sHooks
     {

Framework/KubernetesWorkflow/Location.cs

@@ -1,4 +1,6 @@
-namespace KubernetesWorkflow
+using KubernetesWorkflow.Types;
+
+namespace KubernetesWorkflow
 {
     public interface ILocation
     {

Framework/KubernetesWorkflow/LocationProvider.cs

@@ -1,4 +1,5 @@
-using Logging;
+using KubernetesWorkflow.Types;
+using Logging;
 
 namespace KubernetesWorkflow
 {

Framework/KubernetesWorkflow/LogHandler.cs

@@ -0,0 +1,23 @@
+namespace KubernetesWorkflow
+{
+    public interface ILogHandler
+    {
+        void Log(Stream log);
+    }
+
+    public abstract class LogHandler : ILogHandler
+    {
+        public void Log(Stream log)
+        {
+            using var reader = new StreamReader(log);
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                ProcessLine(line);
+                line = reader.ReadLine();
+            }
+        }
+
+        protected abstract void ProcessLine(string line);
+    }
+}

Framework/KubernetesWorkflow/Recipe/ContainerAdditionals.cs

@@ -1,6 +1,6 @@
 using Newtonsoft.Json.Linq;
 
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class ContainerAdditionals
     {
@@ -20,7 +20,7 @@ namespace KubernetesWorkflow
         {
             var typeName = GetTypeName(typeof(T));
             var userData = Additionals.SingleOrDefault(a => a.Type == typeName);
-            if (userData == null) return default(T);
+            if (userData == null) return default;
             var jobject = (JObject)userData.UserData;
             return jobject.ToObject<T>();
         }

Framework/KubernetesWorkflow/Recipe/ContainerRecipe.cs

@@ -1,4 +1,4 @@
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class ContainerRecipe
     {

Framework/KubernetesWorkflow/Recipe/ContainerRecipeFactory.cs

@@ -1,6 +1,6 @@
 using Utils;
 
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public abstract class ContainerRecipeFactory
     {

Framework/KubernetesWorkflow/Recipe/ContainerResources.cs

@@ -1,6 +1,6 @@
 using Utils;
 
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class ContainerResources
     {

Framework/KubernetesWorkflow/Recipe/PodAnnotations.cs

@@ -1,4 +1,4 @@
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class PodAnnotations
     {

Framework/KubernetesWorkflow/Recipe/PodLabels.cs

@@ -1,4 +1,4 @@
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class PodLabels
     {

Framework/KubernetesWorkflow/Recipe/RecipeComponentFactory.cs

@@ -1,7 +1,7 @@
 using System.Globalization;
 using Utils;
 
-namespace KubernetesWorkflow
+namespace KubernetesWorkflow.Recipe
 {
     public class RecipeComponentFactory
     {

Framework/KubernetesWorkflow/RunningContainers.cs

@@ -1,113 +0,0 @@
-using Logging;
-using Newtonsoft.Json;
-using Utils;
-
-namespace KubernetesWorkflow
-{
-    public class RunningContainers
-    {
-        public RunningContainers(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
-        {
-            StartupConfig = startupConfig;
-            StartResult = startResult;
-            Containers = containers;
-
-            foreach (var c in containers) c.RunningContainers = this;
-        }
-
-        public StartupConfig StartupConfig { get; }
-        public StartResult StartResult { get; }
-        public RunningContainer[] Containers { get; }
-
-        [JsonIgnore]
-        public string Name
-        {
-            get { return $"{Containers.Length}x '{Containers.First().Name}'"; }
-        }
-
-        public string Describe()
-        {
-            return string.Join(",", Containers.Select(c => c.Name));
-        }
-    }
-
-    public class RunningContainer
-    {
-        public RunningContainer(string name, ContainerRecipe recipe, ContainerAddress[] addresses)
-        {
-            Name = name;
-            Recipe = recipe;
-            Addresses = addresses;
-        }
-
-        public string Name { get; }
-        public ContainerRecipe Recipe { get; }
-        public ContainerAddress[] Addresses { get; }
-
-        [JsonIgnore]
-        public RunningContainers RunningContainers { get; internal set; } = null!;
-
-        public Address GetAddress(ILog log, string portTag)
-        {
-            var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
-            if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);
-
-            var select = SelectAddress(addresses);
-            log.Log($"Container '{Name}' selected for tag '{portTag}' address: '{select}'");
-            return select.Address;
-        }
-
-        public Address GetInternalAddress(string portTag)
-        {
-            var containerAddress = Addresses.Single(a => a.PortTag == portTag && a.IsInteral);
-            return containerAddress.Address;
-        }
-
-        private ContainerAddress SelectAddress(ContainerAddress[] addresses)
-        {
-            var location = RunnerLocationUtils.GetRunnerLocation();
-            if (location == RunnerLocation.InternalToCluster)
-            {
-                return addresses.Single(a => a.IsInteral);
-            }
-            if (location == RunnerLocation.ExternalToCluster)
-            {
-                return addresses.Single(a => !a.IsInteral);
-            }
-            throw new Exception("Running location not known.");
-        }
-    }
-
-    public class ContainerAddress
-    {
-        public ContainerAddress(string portTag, Address address, bool isInteral)
-        {
-            PortTag = portTag;
-            Address = address;
-            IsInteral = isInteral;
-        }
-
-        public string PortTag { get; }
-        public Address Address { get; }
-        public bool IsInteral { get; }
-
-        public override string ToString()
-        {
-            var indicator = IsInteral ? "int" : "ext";
-            return $"{indicator} {PortTag} -> '{Address}'";
-        }
-    }
-
-    public static class RunningContainersExtensions
-    {
-        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
-        {
-            return runningContainers.SelectMany(c => c.Containers).ToArray();
-        }
-
-        public static string Describe(this RunningContainers[] runningContainers)
-        {
-            return string.Join(",", runningContainers.Select(c => c.Describe()));
-        }
-    }
-}

Framework/KubernetesWorkflow/RunningPod.cs

@@ -1,5 +1,7 @@
 using k8s;
 using k8s.Models;
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
 using Newtonsoft.Json;
 
 namespace KubernetesWorkflow
@@ -64,76 +66,4 @@ namespace KubernetesWorkflow
             return Array.Empty<Port>();
         }
     }
-
-    public class RunningDeployment
-    {
-        public RunningDeployment(string name, string podLabel)
-        {
-            Name = name;
-            PodLabel = podLabel;
-        }
-
-        public string Name { get; }
-        public string PodLabel { get; }
-
-        public V1Pod GetPod(K8sClient client, string k8sNamespace)
-        {
-            var allPods = client.Run(c => c.ListNamespacedPod(k8sNamespace));
-            var pods = allPods.Items.Where(p => p.GetLabel(K8sController.PodLabelKey) == PodLabel).ToArray();
-
-            if (pods.Length != 1) throw new Exception("Expected to find only 1 pod by podLabel.");
-            return pods[0];
-        }
-    }
-
-    public class RunningService
-    {
-        public RunningService(string name, List<ContainerRecipePortMapEntry> result)
-        {
-            Name = name;
-            Result = result;
-        }
-
-        public string Name { get; }
-        public List<ContainerRecipePortMapEntry> Result { get; }
-
-        public Port? GetServicePortForRecipeAndTag(ContainerRecipe recipe, string tag)
-        {
-            return GetServicePortsForRecipe(recipe).SingleOrDefault(p => p.Tag == tag);
-        }
-
-        public Port[] GetServicePortsForRecipe(ContainerRecipe recipe)
-        {
-            return Result
-                .Where(p => p.RecipeNumber == recipe.Number)
-                .SelectMany(p => p.Ports)
-                .ToArray();
-        }
-    }
-
-    public class ContainerRecipePortMapEntry
-    {
-        public ContainerRecipePortMapEntry(int recipeNumber, Port[] ports)
-        {
-            RecipeNumber = recipeNumber;
-            Ports = ports;
-        }
-
-        public int RecipeNumber { get; }
-        public Port[] Ports { get; }
-    }
-
-    public class PodInfo
-    {
-        public PodInfo(string name, string ip, string k8sNodeName)
-        {
-            Name = name;
-            Ip = ip;
-            K8SNodeName = k8sNodeName;
-        }
-
-        public string Name { get; }
-        public string Ip { get; }
-        public string K8SNodeName { get; }
-    }
 }

Framework/KubernetesWorkflow/StartupWorkflow.cs

@@ -1,4 +1,6 @@
-using Logging;
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
+using Logging;
 using Newtonsoft.Json;
 using Utils;
 
@@ -241,25 +243,4 @@ namespace KubernetesWorkflow
             }
         }
     }
-
-    public interface ILogHandler
-    {
-        void Log(Stream log);
-    }
-
-    public abstract class LogHandler : ILogHandler
-    {
-        public void Log(Stream log)
-        {
-            using var reader = new StreamReader(log);
-            var line = reader.ReadLine();
-            while (line != null)
-            {
-                ProcessLine(line);
-                line = reader.ReadLine();
-            }
-        }
-
-        protected abstract void ProcessLine(string line);
-    }
 }

Framework/KubernetesWorkflow/Types/ContainerAddress.cs

@@ -0,0 +1,24 @@
+using Utils;
+
+namespace KubernetesWorkflow.Types
+{
+    public class ContainerAddress
+    {
+        public ContainerAddress(string portTag, Address address, bool isInteral)
+        {
+            PortTag = portTag;
+            Address = address;
+            IsInteral = isInteral;
+        }
+
+        public string PortTag { get; }
+        public Address Address { get; }
+        public bool IsInteral { get; }
+
+        public override string ToString()
+        {
+            var indicator = IsInteral ? "int" : "ext";
+            return $"{indicator} {PortTag} -> '{Address}'";
+        }
+    }
+}

Framework/KubernetesWorkflow/Types/ContainerRecipePortMapEntry.cs

@@ -0,0 +1,16 @@
+using KubernetesWorkflow.Recipe;
+
+namespace KubernetesWorkflow.Types
+{
+    public class ContainerRecipePortMapEntry
+    {
+        public ContainerRecipePortMapEntry(int recipeNumber, Port[] ports)
+        {
+            RecipeNumber = recipeNumber;
+            Ports = ports;
+        }
+
+        public int RecipeNumber { get; }
+        public Port[] Ports { get; }
+    }
+}

Framework/KubernetesWorkflow/Types/K8sNodeLabel.cs

@@ -0,0 +1,14 @@
+namespace KubernetesWorkflow.Types
+{
+    public class K8sNodeLabel
+    {
+        public K8sNodeLabel(string key, string value)
+        {
+            Key = key;
+            Value = value;
+        }
+
+        public string Key { get; }
+        public string Value { get; }
+    }
+}

Framework/KubernetesWorkflow/Types/PodInfo.cs

@@ -0,0 +1,16 @@
+namespace KubernetesWorkflow.Types
+{
+    public class PodInfo
+    {
+        public PodInfo(string name, string ip, string k8sNodeName)
+        {
+            Name = name;
+            Ip = ip;
+            K8SNodeName = k8sNodeName;
+        }
+
+        public string Name { get; }
+        public string Ip { get; }
+        public string K8SNodeName { get; }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningContainer.cs

@@ -0,0 +1,54 @@
+using KubernetesWorkflow.Recipe;
+using Logging;
+using Newtonsoft.Json;
+using Utils;
+
+namespace KubernetesWorkflow.Types
+{
+    public class RunningContainer
+    {
+        public RunningContainer(string name, ContainerRecipe recipe, ContainerAddress[] addresses)
+        {
+            Name = name;
+            Recipe = recipe;
+            Addresses = addresses;
+        }
+
+        public string Name { get; }
+        public ContainerRecipe Recipe { get; }
+        public ContainerAddress[] Addresses { get; }
+
+        [JsonIgnore]
+        public RunningContainers RunningContainers { get; internal set; } = null!;
+
+        public Address GetAddress(ILog log, string portTag)
+        {
+            var addresses = Addresses.Where(a => a.PortTag == portTag).ToArray();
+            if (!addresses.Any()) throw new Exception("No addresses found for portTag: " + portTag);
+
+            var select = SelectAddress(addresses);
+            log.Debug($"Container '{Name}' selected for tag '{portTag}' address: '{select}'");
+            return select.Address;
+        }
+
+        public Address GetInternalAddress(string portTag)
+        {
+            var containerAddress = Addresses.Single(a => a.PortTag == portTag && a.IsInteral);
+            return containerAddress.Address;
+        }
+
+        private ContainerAddress SelectAddress(ContainerAddress[] addresses)
+        {
+            var location = RunnerLocationUtils.GetRunnerLocation();
+            if (location == RunnerLocation.InternalToCluster)
+            {
+                return addresses.Single(a => a.IsInteral);
+            }
+            if (location == RunnerLocation.ExternalToCluster)
+            {
+                return addresses.Single(a => !a.IsInteral);
+            }
+            throw new Exception("Running location not known.");
+        }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningContainers.cs

@@ -0,0 +1,44 @@
+using Newtonsoft.Json;
+
+namespace KubernetesWorkflow.Types
+{
+    public class RunningContainers
+    {
+        public RunningContainers(StartupConfig startupConfig, StartResult startResult, RunningContainer[] containers)
+        {
+            StartupConfig = startupConfig;
+            StartResult = startResult;
+            Containers = containers;
+
+            foreach (var c in containers) c.RunningContainers = this;
+        }
+
+        public StartupConfig StartupConfig { get; }
+        public StartResult StartResult { get; }
+        public RunningContainer[] Containers { get; }
+
+        [JsonIgnore]
+        public string Name
+        {
+            get { return $"{Containers.Length}x '{Containers.First().Name}'"; }
+        }
+
+        public string Describe()
+        {
+            return string.Join(",", Containers.Select(c => c.Name));
+        }
+    }
+
+    public static class RunningContainersExtensions
+    {
+        public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
+        {
+            return runningContainers.SelectMany(c => c.Containers).ToArray();
+        }
+
+        public static string Describe(this RunningContainers[] runningContainers)
+        {
+            return string.Join(",", runningContainers.Select(c => c.Describe()));
+        }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningDeployment.cs

@@ -0,0 +1,14 @@
+namespace KubernetesWorkflow.Types
+{
+    public class RunningDeployment
+    {
+        public RunningDeployment(string name, string podLabel)
+        {
+            Name = name;
+            PodLabel = podLabel;
+        }
+
+        public string Name { get; }
+        public string PodLabel { get; }
+    }
+}

Framework/KubernetesWorkflow/Types/RunningService.cs

@@ -0,0 +1,29 @@
+using KubernetesWorkflow.Recipe;
+
+namespace KubernetesWorkflow.Types
+{
+    public class RunningService
+    {
+        public RunningService(string name, List<ContainerRecipePortMapEntry> result)
+        {
+            Name = name;
+            Result = result;
+        }
+
+        public string Name { get; }
+        public List<ContainerRecipePortMapEntry> Result { get; }
+
+        public Port? GetServicePortForRecipeAndTag(ContainerRecipe recipe, string tag)
+        {
+            return GetServicePortsForRecipe(recipe).SingleOrDefault(p => p.Tag == tag);
+        }
+
+        public Port[] GetServicePortsForRecipe(ContainerRecipe recipe)
+        {
+            return Result
+                .Where(p => p.RecipeNumber == recipe.Number)
+                .SelectMany(p => p.Ports)
+                .ToArray();
+        }
+    }
+}

Framework/Utils/Time.cs

@@ -56,12 +56,12 @@
 
         public static void Retry(Action action, int maxRetries, string description)
         {
-            Retry(action, maxRetries, TimeSpan.FromSeconds(1), description);
+            Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
         }
 
         public static T Retry<T>(Func<T> action, int maxRetries, string description)
         {
-            return Retry(action, maxRetries, TimeSpan.FromSeconds(1), description);
+            return Retry(action, maxRetries, TimeSpan.FromSeconds(5), description);
         }
 
         public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)

ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs

@@ -1,5 +1,6 @@
 using GethPlugin;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
 using Logging;
 
 namespace CodexContractsPlugin

ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs

@@ -1,6 +1,7 @@
 using Core;
 using GethPlugin;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using Utils;
 

ProjectPlugins/CodexContractsPlugin/ContractsContainerInfoExtractor.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;

ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs

@@ -1,5 +1,6 @@
 using Core;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace CodexDiscordBotPlugin
 {

ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace CodexDiscordBotPlugin
 {

ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
 using Utils;
 
 namespace CodexDiscordBotPlugin

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -1,5 +1,6 @@
 using Core;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Utils;
 
 namespace CodexPlugin
@@ -62,14 +63,17 @@ namespace CodexPlugin
 
         public string UploadFile(FileStream fileStream)
         {
-            // private const string UploadFailedMessage = "Unable to store block";
+            return Http().HttpPostStream("data", fileStream);
-
-            return Http().HttpPostStream("upload", fileStream);
         }
 
         public Stream DownloadFile(string contentId)
         {
-            return Http().HttpGetStream("download/" + contentId);
+            return Http().HttpGetStream("data/" + contentId);
+        }
+
+        public CodexLocalDataResponse[] LocalFiles()
+        {
+            return Http().HttpGetJson<CodexLocalDataResponse[]>("local");
         }
 
         public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)

ProjectPlugins/CodexPlugin/CodexApiTypes.cs

@@ -76,12 +76,7 @@ namespace CodexPlugin
 
         public string peerId { get; set; } = string.Empty;
         public long seqNo { get; set; }
-        public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
+        public string[] addresses { get; set; } = Array.Empty<string>();
-    }
-
-    public class CodexDebugPeerAddressResponse
-    {
-        public string address { get; set; } = string.Empty;
     }
 
     public class CodexDebugThresholdBreaches
@@ -170,4 +165,30 @@ namespace CodexPlugin
     {
         public string cid { get; set; } = string.Empty; 
     }
+
+    public class CodexLocalData
+    {
+        public CodexLocalData(ContentId cid, CodexLocalDataManifestResponse manifest)
+        {
+            Cid = cid;
+            Manifest = manifest;
+        }
+
+        public ContentId Cid { get; }
+        public CodexLocalDataManifestResponse Manifest { get; }
+    }
+
+    public class CodexLocalDataResponse
+    {
+        public string cid { get; set; } = string.Empty;
+        public CodexLocalDataManifestResponse manifest { get; set; } = new();
+    }
+
+    public class CodexLocalDataManifestResponse
+    {
+        public string rootHash { get; set; } = string.Empty;
+        public int originalBytes { get; set; }
+        public int blockSize { get; set; }
+        public bool @protected { get; set; }
+    }
 }

ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs

@@ -1,5 +1,6 @@
 using GethPlugin;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
 using Utils;
 
 namespace CodexPlugin

ProjectPlugins/CodexPlugin/CodexDeployment.cs

@@ -1,6 +1,6 @@
 using CodexContractsPlugin;
 using GethPlugin;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -2,6 +2,7 @@
 using FileUtils;
 using GethPlugin;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using MetricsPlugin;
 using Utils;
@@ -18,6 +19,7 @@ namespace CodexPlugin
         //CodexDebugRepoStoreResponse[] GetDebugRepoStore();
         ContentId UploadFile(TrackedFile file);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
+        CodexLocalData[] LocalFiles();
         void ConnectToPeer(ICodexNode node);
         CodexDebugVersionResponse Version { get; }
         IMarketplaceAccess Marketplace { get; }
@@ -121,6 +123,11 @@ namespace CodexPlugin
             return file;
         }
 
+        public CodexLocalData[] LocalFiles()
+        {
+            return CodexAccess.LocalFiles().Select(l => new CodexLocalData(new ContentId(l.cid), l.manifest)).ToArray();
+        }
+
         public void ConnectToPeer(ICodexNode node)
         {
             var peer = (CodexNode)node;
@@ -158,8 +165,9 @@ namespace CodexPlugin
                 throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}");
             }
 
-            //lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
+            var log = tools.GetLog();
-            //lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
+            log.AddStringReplace(nodePeerId, nodeName);
+            log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
             Version = debugInfo.codex;
         }
 
@@ -205,5 +213,15 @@ namespace CodexPlugin
         }
 
         public string Id { get; }
+
+        public override bool Equals(object? obj)
+        {
+            return obj is ContentId id && Id == id.Id;
+        }
+
+        public override int GetHashCode()
+        {
+            return HashCode.Combine(Id);
+        }
     }
 }

ProjectPlugins/CodexPlugin/CodexNodeFactory.cs

@@ -1,6 +1,7 @@
 using Core;
 using GethPlugin;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {

ProjectPlugins/CodexPlugin/CodexNodeGroup.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using MetricsPlugin;
 using System.Collections;
 

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {
@@ -19,12 +19,13 @@ namespace CodexPlugin
 
         public void Announce()
         {
-            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}'");
+            tools.GetLog().Log($"Loaded with Codex ID: '{codexStarter.GetCodexId()}' - Revision: {codexStarter.GetCodexRevision()}");
         }
 
         public void AddMetadata(IAddMetadata metadata)
         {
             metadata.Add("codexid", codexStarter.GetCodexId());
+            metadata.Add("codexrevision", codexStarter.GetCodexRevision());
         }
 
         public void Decommission()

ProjectPlugins/CodexPlugin/CodexStarter.cs

@@ -1,5 +1,6 @@
 using Core;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 
 namespace CodexPlugin
@@ -65,6 +66,12 @@ namespace CodexPlugin
             return recipe.Image;
         }
 
+        public string GetCodexRevision()
+        {
+            if (versionResponse != null) return versionResponse.revision;
+            return "unknown";
+        }
+
         private StartupConfig CreateStartupConfig(CodexSetup codexSetup)
         {
             var startupConfig = new StartupConfig();

ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace CodexPlugin
 {

ProjectPlugins/DeployAndRunPlugin/CoreInterfaceExtensions.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace DeployAndRunPlugin
 {

ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
 
 namespace DeployAndRunPlugin
 {

ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs

@@ -1,5 +1,6 @@
 using Core;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace DeployAndRunPlugin
 {

ProjectPlugins/GethPlugin/GethContainerInfoExtractor.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using Utils;
 

ProjectPlugins/GethPlugin/GethContainerRecipe.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
 
 namespace GethPlugin
 {

ProjectPlugins/GethPlugin/GethDeployment.cs

@@ -1,5 +1,6 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
 using Newtonsoft.Json;
 
 namespace GethPlugin

ProjectPlugins/GethPlugin/GethNode.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using Nethereum.Contracts;
 using NethereumWorkflow;

ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 
 namespace MetricsPlugin

ProjectPlugins/MetricsPlugin/MetricsAccess.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Utils;
 
 namespace MetricsPlugin

ProjectPlugins/MetricsPlugin/MetricsPlugin.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 
 namespace MetricsPlugin

ProjectPlugins/MetricsPlugin/MetricsQuery.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using System.Globalization;
 

ProjectPlugins/MetricsPlugin/MetricsScrapeTarget.cs

@@ -1,4 +1,4 @@
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace MetricsPlugin
 {

ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs

@@ -1,4 +1,5 @@
 using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
 
 namespace MetricsPlugin
 {

ProjectPlugins/MetricsPlugin/PrometheusStarter.cs

@@ -1,5 +1,6 @@
 using Core;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using System.Text;
 

Tests/CodexContinuousTests/ContinuousTestRunner.cs

@@ -1,4 +1,5 @@
-using DistTestCore.Logs;
+using DistTestCore;
+using DistTestCore.Logs;
 using Logging;
 using Newtonsoft.Json;
 using Utils;
@@ -28,7 +29,7 @@ namespace ContinuousTests
                 new FixtureLog(logConfig, startTime, "Overview"),
                 new ConsoleLog()
             );
-            var statusLog = new StatusLog(logConfig, startTime, "ContinuousTestRun");
+            var statusLog = new StatusLog(logConfig, startTime, "continuous-tests", "ContinuousTestRun");
 
             overviewLog.Log("Initializing...");
 
@@ -43,6 +44,7 @@ namespace ContinuousTests
             var taskFactory = new TaskFactory();
             overviewLog.Log("Startup checks passed. Configuration:");
             overviewLog.Log(JsonConvert.SerializeObject(config, Formatting.Indented));
+            overviewLog.Log("Test framework revision: " + GitInfo.GetStatus());
             overviewLog.Log("Continuous tests starting...");
             overviewLog.Log("");
             var allTests = testFactory.CreateTests();
@@ -57,7 +59,7 @@ namespace ContinuousTests
             }
             else
             {
-                var testLoops = filteredTests.Select(t => new TestLoop(entryPointFactory, taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
+                var testLoops = filteredTests.Select(t => new TestLoop(entryPointFactory, taskFactory, config, overviewLog, statusLog, t.GetType(), t.RunTestEvery, startupChecker, cancelToken)).ToArray();
 
                 foreach (var testLoop in testLoops)
                 {
@@ -109,6 +111,7 @@ namespace ContinuousTests
                     Cancellation.Cts.Cancel();
                     overviewLog.Log($"Congratulations! The targer duration has been reached! ({Time.FormatDuration(targetDuration)})");
                     statusLog.ConcludeTest("Passed", testDuration, testData);
+                    Environment.ExitCode = 0;
                     return;
                 }
             }
@@ -117,6 +120,7 @@ namespace ContinuousTests
                 cancelToken.WaitHandle.WaitOne();
             }
             statusLog.ConcludeTest("Failed", testDuration, testData);
+            Environment.ExitCode = 1;
         }
 
         private Dictionary<string, string> FormatTestRuns(TestLoop[] testLoops)
@@ -124,7 +128,8 @@ namespace ContinuousTests
             var result = new Dictionary<string, string>();
             foreach (var testLoop in testLoops)
             {
-                result.Add($"ctest-{testLoop.Name}", $"passes: {testLoop.NumberOfPasses} - failures: {testLoop.NumberOfFailures}");
+                result.Add("testname", testLoop.Name);
+                result.Add($"summary", $"passes: {testLoop.NumberOfPasses} - failures: {testLoop.NumberOfFailures}");
             }
             return result;
         }

Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs

@@ -1,5 +1,5 @@
 using Core;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using Utils;
 

Tests/CodexContinuousTests/NodeRunner.cs

@@ -1,9 +1,9 @@
-using KubernetesWorkflow;
+using NUnit.Framework;
-using NUnit.Framework;
 using Logging;
 using Utils;
 using Core;
 using CodexPlugin;
+using KubernetesWorkflow.Types;
 
 namespace ContinuousTests
 {

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -1,11 +1,11 @@
 using Logging;
 using Utils;
-using KubernetesWorkflow;
 using NUnit.Framework.Internal;
 using System.Reflection;
 using CodexPlugin;
 using DistTestCore.Logs;
 using Core;
+using KubernetesWorkflow.Types;
 
 namespace ContinuousTests
 {
@@ -16,6 +16,7 @@ namespace ContinuousTests
         private readonly TaskFactory taskFactory;
         private readonly Configuration config;
         private readonly ILog overviewLog;
+        private readonly StatusLog statusLog;
         private readonly TestHandle handle;
         private readonly CancellationToken cancelToken;
         private readonly ICodexNode[] nodes;
@@ -23,11 +24,12 @@ namespace ContinuousTests
         private readonly string testName;
         private static int failureCount = 0;
 
-        public SingleTestRun(EntryPointFactory entryPointFactory, TaskFactory taskFactory, Configuration config, ILog overviewLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken)
+        public SingleTestRun(EntryPointFactory entryPointFactory, TaskFactory taskFactory, Configuration config, ILog overviewLog, StatusLog statusLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken)
         {
             this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
+            this.statusLog = statusLog;
             this.handle = handle;
             this.cancelToken = cancelToken;
             testName = handle.Test.GetType().Name;
@@ -63,13 +65,15 @@ namespace ContinuousTests
         private void RunTest(Action<bool> resultHandler)
         {
             var testStart = DateTime.UtcNow;
+            TimeSpan duration = TimeSpan.Zero;
 
             try
             {
                 RunTestMoments();
+                duration = DateTime.UtcNow - testStart;
 
-                var duration = DateTime.UtcNow - testStart;
                 OverviewLog($" > Test passed. ({Time.FormatDuration(duration)})");
+                UpdateStatusLogPassed(testStart, duration);
 
                 if (!config.KeepPassedTestLogs)
                 {
@@ -81,6 +85,7 @@ namespace ContinuousTests
             {
                 fixtureLog.Error("Test run failed with exception: " + ex);
                 fixtureLog.MarkAsFailed();
+                UpdateStatusLogFailed(testStart, duration, ex.ToString());
 
                 DownloadContainerLogs(testStart);
 
@@ -170,6 +175,25 @@ namespace ContinuousTests
             throw new Exception(exceptionsMessage);
         }
 
+        private void UpdateStatusLogFailed(DateTime testStart, TimeSpan duration, string error)
+        {
+            statusLog.ConcludeTest("Failed", duration, CreateStatusLogData(testStart, error));
+        }
+
+        private void UpdateStatusLogPassed(DateTime testStart, TimeSpan duration)
+        {
+            statusLog.ConcludeTest("Passed", duration, CreateStatusLogData(testStart, "OK"));
+        }
+
+        private Dictionary<string, string> CreateStatusLogData(DateTime testStart, string message)
+        {
+            var result = entryPoint.GetPluginMetadata();
+            result.Add("teststart", testStart.ToString("o"));
+            result.Add("testname", testName);
+            result.Add("message", message);
+            return result;
+        }
+
         private string GetCombinedExceptionsMessage(Exception[] exceptions)
         {
             return string.Join(Environment.NewLine, exceptions.Select(ex => ex.ToString()));

Tests/CodexContinuousTests/TestLoop.cs

@@ -1,4 +1,5 @@
-using Logging;
+using DistTestCore.Logs;
+using Logging;
 
 namespace ContinuousTests
 {
@@ -8,6 +9,7 @@ namespace ContinuousTests
         private readonly TaskFactory taskFactory;
         private readonly Configuration config;
         private readonly ILog overviewLog;
+        private readonly StatusLog statusLog;
         private readonly Type testType;
         private readonly TimeSpan runsEvery;
         private readonly StartupChecker startupChecker;
@@ -15,12 +17,13 @@ namespace ContinuousTests
         private readonly EventWaitHandle runFinishedHandle = new EventWaitHandle(true, EventResetMode.ManualReset);
         private static object testLock = new object();
 
-        public TestLoop(EntryPointFactory entryPointFactory, TaskFactory taskFactory, Configuration config, ILog overviewLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken)
+        public TestLoop(EntryPointFactory entryPointFactory, TaskFactory taskFactory, Configuration config, ILog overviewLog, StatusLog statusLog, Type testType, TimeSpan runsEvery, StartupChecker startupChecker, CancellationToken cancelToken)
         {
             this.entryPointFactory = entryPointFactory;
             this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
+            this.statusLog = statusLog;
             this.testType = testType;
             this.runsEvery = runsEvery;
             this.startupChecker = startupChecker;
@@ -73,7 +76,7 @@ namespace ContinuousTests
         {
             var test = (ContinuousTest)Activator.CreateInstance(testType)!;
             var handle = new TestHandle(test);
-            var run = new SingleTestRun(entryPointFactory, taskFactory, config, overviewLog, handle, startupChecker, cancelToken);
+            var run = new SingleTestRun(entryPointFactory, taskFactory, config, overviewLog, statusLog, handle, startupChecker, cancelToken);
 
             runFinishedHandle.Reset();
             run.Run(runFinishedHandle, result =>

Tests/CodexContinuousTests/Tests/PeersTest.cs

@@ -1,8 +1,9 @@
 using CodexPlugin;
-using DistTestCore.Helpers;
+using CodexTests.Helpers;
+using ContinuousTests;
 using NUnit.Framework;
 
-namespace ContinuousTests.Tests
+namespace CodexContinuousTests.Tests
 {
     public class PeersTest : ContinuousTest
     {

Tests/CodexTests/BasicTests/ContinuousSubstitute.cs

@@ -1,7 +1,7 @@
 using CodexContractsPlugin;
 using CodexPlugin;
 using GethPlugin;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using MetricsPlugin;
 using NUnit.Framework;

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -16,7 +16,10 @@ namespace CodexTests.BasicTests
         {
             var primary = AddCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn)));
 
-            primary.UploadFile(GenerateTestFile(5.MB()));
+            var cid = primary.UploadFile(GenerateTestFile(5.MB()));
+
+            var content = primary.LocalFiles();
+            CollectionAssert.Contains(content.Select(c => c.Cid), cid);
 
             var log = Ci.DownloadLog(primary);
 

Tests/CodexTests/CodexDistTest.cs

@@ -1,11 +1,11 @@
 using CodexContractsPlugin;
 using CodexNetDeployer;
 using CodexPlugin;
+using CodexTests.Helpers;
 using Core;
 using DistTestCore;
 using DistTestCore.Helpers;
 using DistTestCore.Logs;
-using GethPlugin;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
 

Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -1,10 +1,9 @@
 using CodexContractsPlugin;
-using CodexTests;
 using GethPlugin;
 using NUnit.Framework;
 using Utils;
 
-namespace Tests.DownloadConnectivityTests
+namespace CodexTests.DownloadConnectivityTests
 {
     [TestFixture]
     public class FullyConnectedDownloadTests : AutoBootstrapDistTest

Tests/CodexTests/Helpers/FullConnectivityHelper.cs

@@ -2,7 +2,7 @@
 using Logging;
 using NUnit.Framework;
 
-namespace DistTestCore.Helpers
+namespace CodexTests.Helpers
 {
     public interface IFullConnectivityImplementation
     {

Tests/CodexTests/Helpers/PeerConnectionTestHelpers.cs

@@ -1,8 +1,8 @@
 using CodexPlugin;
 using Logging;
-using static DistTestCore.Helpers.FullConnectivityHelper;
+using static CodexTests.Helpers.FullConnectivityHelper;
 
-namespace DistTestCore.Helpers
+namespace CodexTests.Helpers
 {
     public class PeerConnectionTestHelpers : IFullConnectivityImplementation
     {

Tests/CodexTests/Helpers/PeerDownloadTestHelpers.cs

@@ -2,9 +2,9 @@
 using FileUtils;
 using Logging;
 using Utils;
-using static DistTestCore.Helpers.FullConnectivityHelper;
+using static CodexTests.Helpers.FullConnectivityHelper;
 
-namespace DistTestCore.Helpers
+namespace CodexTests.Helpers
 {
     public class PeerDownloadTestHelpers : IFullConnectivityImplementation
     {

Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs

@@ -1,8 +1,7 @@
 using CodexPlugin;
-using CodexTests;
 using NUnit.Framework;
 
-namespace Tests.PeerDiscoveryTests
+namespace CodexTests.PeerDiscoveryTests
 {
     [TestFixture]
     public class LayeredDiscoveryTests : CodexDistTest

Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -1,9 +1,9 @@
 using CodexContractsPlugin;
-using CodexTests;
+using CodexPlugin;
 using GethPlugin;
 using NUnit.Framework;
 
-namespace Tests.PeerDiscoveryTests
+namespace CodexTests.PeerDiscoveryTests
 {
     [TestFixture]
     public class PeerDiscoveryTests : AutoBootstrapDistTest
@@ -49,7 +49,45 @@ namespace Tests.PeerDiscoveryTests
 
         private void AssertAllNodesConnected()
         {
-            CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes());
+            var allNodes = GetAllOnlineCodexNodes();
+            CreatePeerConnectionTestHelpers().AssertFullyConnected(allNodes);
+            CheckRoutingTable(allNodes);
+        }
+
+        private void CheckRoutingTable(IEnumerable<ICodexNode> allNodes)
+        {
+            var allResponses = allNodes.Select(n => n.GetDebugInfo()).ToArray();
+
+            var errors = new List<string>();
+            foreach (var response in allResponses)
+            {
+                var error = AreAllPresent(response, allResponses);
+                if (!string.IsNullOrEmpty(error)) errors.Add(error);
+            }
+
+            if (errors.Any())
+            {
+                Assert.Fail(string.Join(Environment.NewLine, errors));
+            }
+        }
+
+        private string AreAllPresent(CodexDebugResponse info, CodexDebugResponse[] allResponses)
+        {
+            var knownIds = info.table.nodes.Select(n => n.nodeId).ToArray();
+            var allOthers = GetAllOtherResponses(info, allResponses);
+            var expectedIds = allOthers.Select(i => i.table.localNode.nodeId).ToArray();
+
+            if (!expectedIds.All(ex => knownIds.Contains(ex)))
+            {
+                return $"Node {info.id}: Not all of '{string.Join(",", expectedIds)}' were present in routing table: '{string.Join(",", knownIds)}'";
+            }
+
+            return string.Empty;
+        }
+
+        private CodexDebugResponse[] GetAllOtherResponses(CodexDebugResponse exclude, CodexDebugResponse[] allResponses)
+        {
+            return allResponses.Where(r => r.id != exclude.id).ToArray();
         }
     }
 }

Tests/DistTestCore/DistTest.cs

@@ -29,7 +29,7 @@ namespace DistTestCore
             var logConfig = configuration.GetLogConfig();
             var startTime = DateTime.UtcNow;
             fixtureLog = new FixtureLog(logConfig, startTime);
-            statusLog = new StatusLog(logConfig, startTime);
+            statusLog = new StatusLog(logConfig, startTime, "dist-tests");
 
             globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder());
 
@@ -58,6 +58,7 @@ namespace DistTestCore
                 throw;
             }
 
+            fixtureLog.Log("Test framework revision: " + GitInfo.GetStatus());
             fixtureLog.Log("Global setup cleanup successful");
         }
 

Tests/DistTestCore/DistTestCore.csproj

@@ -8,6 +8,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+	  <PackageReference Include="LibGit2Sharp" Version="0.28.0" />
 	  <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
 	  <PackageReference Include="nunit" Version="3.13.3" />
 	  <PackageReference Include="NUnit3TestAdapter" Version="4.4.2" />

Tests/DistTestCore/GitInfo.cs

@@ -0,0 +1,39 @@
+using KubernetesWorkflow;
+using LibGit2Sharp;
+using System.Reflection;
+
+namespace DistTestCore
+{
+    public static class GitInfo
+    {
+        private static string? status = null;
+
+        public static string GetStatus()
+        {
+            if (status == null) status = DetermineStatus();
+            return status;
+        }
+
+        private static string DetermineStatus()
+        {
+            var path = FindGitPath();
+            if (path == null) return "unknown";
+
+            using var repo = new Repository(path);
+            var sha = repo.Head.Tip.Sha.Substring(0, 7);
+
+            return K8sNameUtils.Format(sha);
+        }
+
+        private static string? FindGitPath()
+        {
+            var path = Repository.Discover(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
+            if (!string.IsNullOrEmpty(path)) return path;
+                
+            path = Repository.Discover(Directory.GetCurrentDirectory());
+            if (!string.IsNullOrEmpty(path)) return path;
+
+            return null;
+        }
+    }
+}

Tests/DistTestCore/Logs/StatusLog.cs

@@ -1,5 +1,6 @@
 using Logging;
 using Newtonsoft.Json;
+using Utils;
 
 namespace DistTestCore.Logs
 {
@@ -8,11 +9,18 @@ namespace DistTestCore.Logs
         private readonly object fileLock = new object();
         private readonly string fullName;
         private readonly string fixtureName;
+        private readonly string testType;
 
-        public StatusLog(LogConfig config, DateTime start, string name = "")
+        public StatusLog(LogConfig config, DateTime start, string testType, string name = "")
         {
             fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log";
             fixtureName = NameUtils.GetRawFixtureName();
+            this.testType = testType;
+        }
+
+        public void ConcludeTest(string resultStatus, TimeSpan testDuration, Dictionary<string, string> data)
+        {
+            ConcludeTest(resultStatus, Time.FormatDuration(testDuration), data);
         }
 
         public void ConcludeTest(string resultStatus, string testDuration, Dictionary<string, string> data)
@@ -20,11 +28,13 @@ namespace DistTestCore.Logs
             data.Add("timestamp", DateTime.UtcNow.ToString("o"));
             data.Add("runid", NameUtils.GetRunId());
             data.Add("status", resultStatus);
-            data.Add("testid", NameUtils.GetTestId());
             data.Add("category", NameUtils.GetCategoryName());
             data.Add("fixturename", fixtureName);
-            data.Add("testname", NameUtils.GetTestMethodName());
+            if (!data.ContainsKey("testname")) data.Add("testname", NameUtils.GetTestMethodName());
+            data.Add("testid", NameUtils.GetTestId());
+            data.Add("testtype", testType);
             data.Add("testduration", testDuration);
+            data.Add("testframeworkrevision", GitInfo.GetStatus());
             Write(data);
         }
 

Tests/DistTestCore/TestLifecycle.cs

@@ -2,6 +2,8 @@
 using DistTestCore.Logs;
 using FileUtils;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
 using Utils;
 
 namespace DistTestCore
@@ -80,6 +82,7 @@ namespace DistTestCore
             recipe.PodLabels.Add("category", NameUtils.GetCategoryName());
             recipe.PodLabels.Add("fixturename", NameUtils.GetRawFixtureName());
             recipe.PodLabels.Add("testname", NameUtils.GetTestMethodName());
+            recipe.PodLabels.Add("testframeworkrevision", GitInfo.GetStatus());
 
             foreach (var pair in metadata)
             {

Tools/CodexNetDeployer/Deployer.cs

@@ -3,7 +3,7 @@ using CodexDiscordBotPlugin;
 using CodexPlugin;
 using Core;
 using GethPlugin;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using MetricsPlugin;
 

Tools/CodexNetDeployer/K8sHook.cs

@@ -1,5 +1,7 @@
 using DistTestCore;
 using KubernetesWorkflow;
+using KubernetesWorkflow.Recipe;
+using KubernetesWorkflow.Types;
 
 namespace CodexNetDeployer
 {

Tools/CodexNetDeployer/PeerConnectivityChecker.cs

@@ -1,4 +1,4 @@
-using DistTestCore.Helpers;
+using CodexTests.Helpers;
 using Logging;
 
 namespace CodexNetDeployer

Tools/TestClusterStarter/ClusterTestSpec.cs

@@ -1,4 +1,4 @@
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 
 namespace TestClusterStarter
 {

Tools/TestClusterStarter/Program.cs

@@ -1,7 +1,7 @@
 using ArgsUniform;
 using Core;
 using DeployAndRunPlugin;
-using KubernetesWorkflow;
+using KubernetesWorkflow.Types;
 using Logging;
 using Newtonsoft.Json;
 using TestClusterStarter;

docker/continuous-tests-job.yaml

@@ -30,7 +30,7 @@ spec:
         - name: KUBECONFIG
           value: "/opt/kubeconfig.yaml"
         - name: LOGPATH
-          value: "/var/log/codex-continuous-tests"
+          value: "/var/log/codex-continuous-tests/${DEPLOYMENT_NAMESPACE}"
         - name: NAMESPACE
           value: "${NAMESPACE}"
         - name: BRANCH