Merge branch 'master' into feature/docker-image-testruns

# Conflicts:
#	DistTestCore/Codex/CodexContainerRecipe.cs

.github/workflows/dist-tests.yaml

@@ -15,19 +15,19 @@ on:
   workflow_dispatch:
     inputs:
       branch:
-        description: Branch
+        description: Branch (master)
         required: false
         type: string
       source:
-        description: Repository with tests
+        description: Repository with tests (current)
         required: false
         type: string
       nameprefix:
-        description: Runner job/pod name prefix
+        description: Runner prefix (cs-codex-dist-tests)
         required: false
         type: string
       namespace:
-        description: Kubernetes namespace for runner
+        description: Runner namespace (cs-codex-dist-tests)
         required: false
         type: string
 
@@ -56,6 +56,8 @@ jobs:
           [[ -n "${{ inputs.source }}" ]] && echo "SOURCE=${{ inputs.source }}" >>"$GITHUB_ENV" || echo "SOURCE=${{ env.SOURCE }}" >>"$GITHUB_ENV"
           [[ -n "${{ inputs.nameprefix }}" ]] && echo "NAMEPREFIX=${{ inputs.nameprefix }}" >>"$GITHUB_ENV" || echo "NAMEPREFIX=${{ env.NAMEPREFIX }}" >>"$GITHUB_ENV"
           [[ -n "${{ inputs.namespace }}" ]] && echo "NAMESPACE=${{ inputs.namespace }}" >>"$GITHUB_ENV" || echo "NAMESPACE=${{ env.NAMESPACE }}" >>"$GITHUB_ENV"
+          echo "RUNID=$(date +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
+          echo "TESTID=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
 
       - name: Kubectl - Install ${{ env.KUBE_VERSION }}
         uses: azure/setup-kubectl@v3
@@ -69,5 +71,4 @@ jobs:
 
       - name: Kubectl - Create Job
         run: |
-          export RUNID=$(date +%Y%m%d-%H%M%S)
           envsubst < ${{ env.JOB_MANIFEST }} | kubectl apply -f -

CodexNetDeployer/Program.cs

@@ -1,6 +1,9 @@
 using ArgsUniform;
 using CodexNetDeployer;
 using DistTestCore;
+using DistTestCore.Codex;
+using DistTestCore.Marketplace;
+using DistTestCore.Metrics;
 using Newtonsoft.Json;
 using Configuration = CodexNetDeployer.Configuration;
 
@@ -29,6 +32,12 @@ public class Program
             return;
         }
 
+        Console.WriteLine("Using images:" + nl +
+            $"\tCodex image: '{CodexContainerRecipe.DockerImage}'" + nl +
+            $"\tCodex Contracts image: '{CodexContractsContainerRecipe.DockerImage}'" + nl +
+            $"\tPrometheus image: '{PrometheusContainerRecipe.DockerImage}'" + nl +
+            $"\tGeth image: '{GethContainerRecipe.DockerImage}'" + nl);
+
         if (!args.Any(a => a == "-y"))
         {
             Console.WriteLine("Does the above config look good? [y/n]");

CodexNetDeployer/deploy-continuous-testnet.sh

@@ -3,10 +3,11 @@ dotnet run \
     --kube-namespace=codex-continuous-tests \
     --nodes=5 \
     --validators=3 \
+    --log-level=Trace \
     --storage-quota=2048 \
     --storage-sell=1024 \
     --min-price=1024 \
     --max-collateral=1024 \
     --max-duration=3600000 \
-    --block-ttl=120
+    --block-ttl=120 \
-
+	-y

ContinuousTests/Configuration.cs

@@ -25,6 +25,9 @@ namespace ContinuousTests
         [Uniform("stop", "s", "STOPONFAIL", false, "If true, runner will stop on first test failure and download all cluster container logs. False by default.")]
         public bool StopOnFailure { get; set; } = false;
 
+        [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
+        public bool DownloadContainerLogs { get; set; } = false;
+
         public CodexDeployment CodexDeployment { get; set; } = null!;
 
         public TestRunnerLocation RunnerLocation { get; set; } = TestRunnerLocation.InternalToCluster;
@@ -57,9 +60,11 @@ namespace ContinuousTests
         private static void PrintHelp()
         {
             var nl = Environment.NewLine;
-            Console.WriteLine("CodexNetDownloader lets you download all container logs given a codex-deployment.json file." + nl);
+            Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl +
+                "The tests will run in an endless loop unless otherwise specified, using the test-specific timing values." + nl);
 
-            Console.WriteLine("CodexNetDownloader assumes you are running this tool from *inside* the Kubernetes cluster. " +
+
+            Console.WriteLine("ContinuousTests assumes you are running this tool from *inside* the Kubernetes cluster. " +
                 "If you are not running this from a container inside the cluster, add the argument '--external'." + nl);
         }
     }

ContinuousTests/ContinuousLogDownloader.cs

@@ -0,0 +1,99 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
+
+namespace ContinuousTests
+{
+    public class ContinuousLogDownloader
+    {
+        private readonly TestLifecycle lifecycle;
+        private readonly CodexDeployment deployment;
+        private readonly string outputPath;
+        private readonly CancellationToken cancelToken;
+
+        public ContinuousLogDownloader(TestLifecycle lifecycle, CodexDeployment deployment, string outputPath, CancellationToken cancelToken)
+        {
+            this.lifecycle = lifecycle;
+            this.deployment = deployment;
+            this.outputPath = outputPath;
+            this.cancelToken = cancelToken;
+        }
+
+        public void Run()
+        {
+            while (!cancelToken.IsCancellationRequested)
+            {
+                UpdateLogs();
+
+                cancelToken.WaitHandle.WaitOne(TimeSpan.FromSeconds(15));
+            }
+
+            // After testing has stopped, we wait a little bit and fetch the logs one more time.
+            // If our latest fetch was not recent, interesting test-related log activity might
+            // not have been captured yet.
+            Thread.Sleep(TimeSpan.FromSeconds(10));
+            UpdateLogs();
+        }
+
+        private void UpdateLogs()
+        {
+            foreach (var container in deployment.CodexContainers)
+            {
+                UpdateLog(container);
+            }
+        }
+
+        private void UpdateLog(RunningContainer container)
+        {
+            var filepath = Path.Combine(outputPath, GetLogName(container));
+            if (!File.Exists(filepath))
+            {
+                File.WriteAllLines(filepath, new[] { container.Name });
+            }
+
+            var appender = new LogAppender(filepath);
+
+            lifecycle.CodexStarter.DownloadLog(container, appender);
+        }
+
+        private static string GetLogName(RunningContainer container)
+        {
+            return container.Name
+                .Replace("<", "")
+                .Replace(">", "")
+                + ".log";
+        }
+    }
+
+    public class LogAppender : ILogHandler
+    {
+        private readonly string filename;
+
+        public LogAppender(string filename)
+        {
+            this.filename = filename;
+        }
+
+        public void Log(Stream log)
+        {
+            using var reader = new StreamReader(log);
+            var lines = File.ReadAllLines(filename);
+            var lastLine = lines.Last();
+            var recording = lines.Length < 3;
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                if (recording)
+                {
+                    File.AppendAllLines(filename, new[] { line });
+                }
+                else
+                {
+                    recording = line == lastLine;
+                }
+
+                line = reader.ReadLine();
+            }
+        }
+    }
+}

ContinuousTests/ContinuousTestRunner.cs

@@ -24,12 +24,14 @@ namespace ContinuousTests
             startupChecker.Check();
 
             var taskFactory = new TaskFactory();
-            var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
+            var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "Overview");
             overviewLog.Log("Continuous tests starting...");
             var allTests = testFactory.CreateTests();
 
             ClearAllCustomNamespaces(allTests, overviewLog);
 
+            StartLogDownloader(taskFactory);
+
             var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
 
             foreach (var testLoop in testLoops)
@@ -61,5 +63,18 @@ namespace ContinuousTests
             var (workflowCreator, _) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, test.CustomK8sNamespace, new DefaultTimeSet(), log, config.RunnerLocation);
             workflowCreator.CreateWorkflow().DeleteTestResources();
         }
+
+        private void StartLogDownloader(TaskFactory taskFactory)
+        {
+            if (!config.DownloadContainerLogs) return;
+
+            var path = Path.Combine(config.LogPath, "containers");
+            if (!Directory.Exists(path)) Directory.CreateDirectory(path);
+
+            var (_, lifecycle) = k8SFactory.CreateFacilities(config.KubeConfigFile, config.LogPath, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), new NullLog(), config.RunnerLocation);
+            var downloader = new ContinuousLogDownloader(lifecycle, config.CodexDeployment, path, cancelToken);
+
+            taskFactory.Run(downloader.Run);
+        }
     }
 }

ContinuousTests/SingleTestRun.cs

@@ -32,7 +32,7 @@ namespace ContinuousTests
             this.handle = handle;
             this.cancelToken = cancelToken;
             testName = handle.Test.GetType().Name;
-            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), testName);
+            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
 
             nodes = CreateRandomNodes(handle.Test.RequiredNumberOfNodes);
             dataFolder = config.DataPath + "-" + Guid.NewGuid();

ContinuousTests/StartupChecker.cs

@@ -19,7 +19,7 @@ namespace ContinuousTests
 
         public void Check()
         {
-            var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
+            var log = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, "StartupChecks");
             log.Log("Starting continuous test run...");
             log.Log("Checking configuration...");
             PreflightCheck(config);

ContinuousTests/run.sh

@@ -1,4 +1,6 @@
 dotnet run \
     --kube-config=/opt/kubeconfig.yaml \
     --codex-deployment=codex-deployment.json \
-    --stop=1 
+    --keep=1 \
+    --stop=1 \
+    --dl-logs=1

DistTestCore/Codex/CodexAccess.cs

@@ -22,17 +22,12 @@ namespace DistTestCore.Codex
 
         public CodexDebugResponse GetDebugInfo()
         {
-            return Http(TimeSpan.FromSeconds(60)).HttpGetJson<CodexDebugResponse>("debug/info");
+            return Http().HttpGetJson<CodexDebugResponse>("debug/info");
         }
 
         public CodexDebugPeerResponse GetDebugPeer(string peerId)
         {
-            return GetDebugPeer(peerId, TimeSpan.FromSeconds(10));
+            var http = Http();
-        }
-
-        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
-        {
-            var http = Http(timeout);
             var str = http.HttpGetString($"debug/peer/{peerId}");
 
             if (str.ToLowerInvariant() == "unable to find peer!")
@@ -50,7 +45,8 @@ namespace DistTestCore.Codex
 
         public int GetDebugFutures()
         {
-            return Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
+            // Some Codex images support debug/futures to count the number of open futures.
+            return 0; // Http().HttpGetJson<CodexDebugFutures>("debug/futures").futures;
         }
 
         public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
@@ -88,9 +84,9 @@ namespace DistTestCore.Codex
             return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
         }
 
-        private Http Http(TimeSpan? timeoutOverride = null)
+        private Http Http()
         {
-            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name, timeoutOverride);
+            return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name);
         }
     }
 }

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -5,6 +5,8 @@ namespace DistTestCore.Codex
 {
     public class CodexContainerRecipe : ContainerRecipeFactory
     {
+        private const string DefaultDockerImage = "codexstorage/nim-codex:sha-14c5270";
+
         public const string MetricsPortTag = "metrics_port";
         public const string DiscoveryPortTag = "discovery-port";
 
@@ -87,7 +89,7 @@ namespace DistTestCore.Codex
         {
             var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE");
             if (!string.IsNullOrEmpty(image)) return image;
-            return "codexstorage/nim-codex:sha-0265cad";
+            return DefaultDockerImage;
         }
     }
 }

DistTestCore/DistTest.cs

@@ -16,6 +16,7 @@ namespace DistTestCore
         private readonly Configuration configuration = new Configuration();
         private readonly Assembly[] testAssemblies;
         private readonly FixtureLog fixtureLog;
+        private readonly StatusLog statusLog;
         private readonly object lifecycleLock = new object();
         private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
 
@@ -24,7 +25,10 @@ namespace DistTestCore
             var assemblies = AppDomain.CurrentDomain.GetAssemblies();
             testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
 
-            fixtureLog = new FixtureLog(configuration.GetLogConfig());
+            var logConfig = configuration.GetLogConfig();
+            var startTime = DateTime.UtcNow;
+            fixtureLog = new FixtureLog(logConfig, startTime);
+            statusLog = new StatusLog(logConfig, startTime, CodexContainerRecipe.DefaultDockerImage);
 
             PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
             PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
@@ -186,6 +190,7 @@ namespace DistTestCore
         private void CreateNewTestLifecycle()
         {
             var testName = GetCurrentTestName();
+            fixtureLog.WriteLogTag();
             Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
             {
                 lock (lifecycleLock)
@@ -198,7 +203,10 @@ namespace DistTestCore
         private void DisposeTestLifecycle()
         {
             var lifecycle = Get();
-            fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
+            var testResult = GetTestResult();
+            var testDuration = lifecycle.GetTestDuration();
+            fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
+            statusLog.ConcludeTest(testResult, testDuration);
             Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
             {
                 lifecycle.Log.EndTest();

DistTestCore/Helpers/FullConnectivityHelper.cs

@@ -0,0 +1,211 @@
+using DistTestCore.Codex;
+using NUnit.Framework;
+using Utils;
+
+namespace DistTestCore.Helpers
+{
+    public interface IFullConnectivityImplementation
+    {
+        string Description();
+        string ValidateEntry(FullConnectivityHelper.Entry entry, FullConnectivityHelper.Entry[] allEntries);
+        FullConnectivityHelper.PeerConnectionState Check(FullConnectivityHelper.Entry from, FullConnectivityHelper.Entry to);
+    }
+
+    public class FullConnectivityHelper
+    {
+        private static string Nl = Environment.NewLine;
+        private readonly DistTest test;
+        private readonly IFullConnectivityImplementation implementation;
+
+        public FullConnectivityHelper(DistTest test, IFullConnectivityImplementation implementation)
+        {
+            this.test = test;
+            this.implementation = implementation;
+        }
+
+        public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullyConnected(nodes.ToArray());
+        }
+
+        private void AssertFullyConnected(IOnlineCodexNode[] nodes)
+        {
+            test.Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            var entries = CreateEntries(nodes);
+            var pairs = CreatePairs(entries);
+
+            RetryWhilePairs(pairs, () =>
+            {
+                CheckAndRemoveSuccessful(pairs);
+            });
+
+            if (pairs.Any())
+            {
+                var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()));
+
+                test.Log($"Connections failed:{Nl}{pairDetails}");
+
+                Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())));
+            }
+            else
+            {
+                test.Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
+            }
+        }
+
+        private static void RetryWhilePairs(List<Pair> pairs, Action action)
+        {
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
+            while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow)
+            {
+                action();
+
+                Time.Sleep(TimeSpan.FromSeconds(2));
+            }
+        }
+
+        private void CheckAndRemoveSuccessful(List<Pair> pairs)
+        {
+            // For large sets, don't try and do all of them at once.
+            var selectedPair = pairs.Take(20).ToArray();
+            var pairDetails = new List<string>();
+
+            foreach (var pair in selectedPair)
+            {
+                test.ScopedTestFiles(pair.Check);
+
+                if (pair.Success)
+                {
+                    pairDetails.AddRange(pair.GetResultMessages());
+                    pairs.Remove(pair);
+                }
+            }
+
+            test.Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}");
+        }
+
+        private Entry[] CreateEntries(IOnlineCodexNode[] nodes)
+        {
+            var entries = nodes.Select(n => new Entry(n)).ToArray();
+
+            var errors = entries
+                            .Select(e => implementation.ValidateEntry(e, entries))
+                            .Where(s => !string.IsNullOrEmpty(s))
+                            .ToArray();
+
+            if (errors.Any())
+            {
+                Assert.Fail("Some node entries failed to validate: " + string.Join(Nl, errors));
+            }
+
+            return entries;
+        }
+
+        private List<Pair> CreatePairs(Entry[] entries)
+        {
+            return CreatePairsIterator(entries).ToList();
+        }
+
+        private IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
+        {
+            for (var x = 0; x < entries.Length; x++)
+            {
+                for (var y = x + 1; y < entries.Length; y++)
+                {
+                    yield return new Pair(implementation, entries[x], entries[y]);
+                }
+            }
+        }
+
+        public class Entry
+        {
+            public Entry(IOnlineCodexNode node)
+            {
+                Node = node;
+                Response = node.GetDebugInfo();
+            }
+
+            public IOnlineCodexNode Node { get; }
+            public CodexDebugResponse Response { get; }
+
+            public override string ToString()
+            {
+                if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
+                return Response.id;
+            }
+        }
+
+        public enum PeerConnectionState
+        {
+            Unknown,
+            Connection,
+            NoConnection,
+        }
+
+        public class Pair
+        {
+            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
+            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
+            private readonly IFullConnectivityImplementation implementation;
+
+            public Pair(IFullConnectivityImplementation implementation, Entry a, Entry b)
+            {
+                this.implementation = implementation;
+                A = a;
+                B = b;
+            }
+
+            public Entry A { get; }
+            public Entry B { get; }
+            public PeerConnectionState AKnowsB { get; private set; }
+            public PeerConnectionState BKnowsA { get; private set; }
+            public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
+            public bool Inconclusive { get { return AKnowsB == PeerConnectionState.Unknown || BKnowsA == PeerConnectionState.Unknown; } }
+
+            public void Check()
+            {
+                aToBTime = Measure(() => AKnowsB = Check(A, B));
+                bToATime = Measure(() => BKnowsA = Check(B, A));
+            }
+
+            public override string ToString()
+            {
+                return $"[{string.Join(",", GetResultMessages())}]";
+            }
+
+            public string[] GetResultMessages()
+            {
+                var aName = A.ToString();
+                var bName = B.ToString();
+
+                return new[]
+                {
+                    $"[{aName} --> {bName}] = {AKnowsB} ({aToBTime.TotalSeconds} seconds)",
+                    $"[{aName} <-- {bName}] = {BKnowsA} ({bToATime.TotalSeconds} seconds)"
+                };
+            }
+
+            private static TimeSpan Measure(Action action)
+            {
+                var start = DateTime.UtcNow;
+                action();
+                return DateTime.UtcNow - start;
+            }
+
+            private PeerConnectionState Check(Entry from, Entry to)
+            {
+                Thread.Sleep(10);
+
+                try
+                {
+                    return implementation.Check(from, to);
+                }
+                catch
+                {
+                    // Didn't get a conclusive answer. Try again later.
+                    return PeerConnectionState.Unknown;
+                }
+            }
+        }
+    }
+}

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -1,149 +1,55 @@
 using DistTestCore.Codex;
-using NUnit.Framework;
+using static DistTestCore.Helpers.FullConnectivityHelper;
-using Utils;
 
 namespace DistTestCore.Helpers
 {
-    public class PeerConnectionTestHelpers
+    public class PeerConnectionTestHelpers : IFullConnectivityImplementation
     {
-        private readonly Random random = new Random();
+        private readonly FullConnectivityHelper helper;
-        private readonly DistTest test;
 
         public PeerConnectionTestHelpers(DistTest test)
         {
-            this.test = test;
+            helper = new FullConnectivityHelper(test, this);
         }
 
         public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
         {
-            var n = nodes.ToArray();
+            helper.AssertFullyConnected(nodes);
-
-            AssertFullyConnected(n);
-
-            for (int i = 0; i < 5; i++)
-            {
-                Time.Sleep(TimeSpan.FromSeconds(30));
-                AssertFullyConnected(n);
-            }
         }
 
-        private void AssertFullyConnected(IOnlineCodexNode[] nodes)
+        public string Description()
         {
-            test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            return "Peer Discovery";
-            var entries = CreateEntries(nodes);
-            var pairs = CreatePairs(entries);
-
-            RetryWhilePairs(pairs, () =>
-            {
-                CheckAndRemoveSuccessful(pairs);
-            });
-
-            if (pairs.Any())
-            {
-                test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
-                Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
-                test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
-            }
-            else
-            {
-                test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
-            }
         }
 
-        private static void RetryWhilePairs(List<Pair> pairs, Action action)
+        public string ValidateEntry(Entry entry, Entry[] allEntries)
         {
-            var timeout = DateTime.UtcNow + TimeSpan.FromSeconds(30);
+            var result = string.Empty;
-            while (pairs.Any() && timeout > DateTime.UtcNow)
+            foreach (var peer in entry.Response.table.nodes)
-            {
-                action();
-
-                if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(2));
-            }
-        }
-
-        private void CheckAndRemoveSuccessful(List<Pair> pairs)
-        {
-            var checkTasks = pairs.Select(p => Task.Run(() =>
-            {
-                ApplyRandomDelay();
-                p.Check();
-            })).ToArray();
-
-            Task.WaitAll(checkTasks);
-
-            foreach (var pair in pairs.ToArray())
-            {
-                if (pair.Success)
-                {
-                    test.Log(pair.GetMessage());
-                    pairs.Remove(pair);
-                }
-            }
-        }
-
-        private static Entry[] CreateEntries(IOnlineCodexNode[] nodes)
-        {
-            var entries = nodes.Select(n => new Entry(n)).ToArray();
-            var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
-
-            if (incorrectDiscoveryEndpoints.Any())
-            {
-                Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
-                    string.Join(Environment.NewLine, incorrectDiscoveryEndpoints));
-            }
-
-            return entries;
-        }
-
-        private static List<Pair> CreatePairs(Entry[] entries)
-        {
-            return CreatePairsIterator(entries).ToList();
-        }
-
-        private static IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
-        {
-            for (var x = 0; x < entries.Length; x++)
-            {
-                for (var y = x + 1; y < entries.Length; y++)
-                {
-                    yield return new Pair(entries[x], entries[y]);
-                }
-            }
-        }
-
-        private void ApplyRandomDelay()
-        {
-            // Calling all the nodes all at the same time is not exactly nice.
-            Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000)));
-        }
-
-        public class Entry
-        {
-            public Entry(IOnlineCodexNode node)
-            {
-                Node = node;
-                Response = node.GetDebugInfo();
-            }
-
-            public IOnlineCodexNode Node { get; }
-            public CodexDebugResponse Response { get; }
-
-            public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
-            {
-                foreach (var peer in Response.table.nodes)
             {
                 var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
                 if (expected != peer.address)
                 {
-                        yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'";
+                    result += $"Node:{entry.Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'. ";
                 }
             }
+            return result;
         }
 
-            public override string ToString()
+        public PeerConnectionState Check(Entry from, Entry to)
         {
-                if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN";
+            var peerId = to.Response.id;
-                return Response.id;
+
+            var response = from.Node.GetDebugPeer(peerId);
+            if (!response.IsPeerFound)
+            {
+                return PeerConnectionState.NoConnection;
+            }
+            if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
+            {
+                return PeerConnectionState.Connection;
+            }
+            return PeerConnectionState.Unknown;
         }
 
         private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
@@ -157,102 +63,4 @@ namespace DistTestCore.Helpers
             return $"{ip}:{discPort.Number}";
         }
     }
-
-        public enum PeerConnectionState
-        {
-            Unknown,
-            Connection,
-            NoConnection,
-        }
-
-        public class Pair
-        {
-            private readonly TimeSpan timeout = TimeSpan.FromSeconds(60);
-            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
-            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
-
-            public Pair(Entry a, Entry b)
-            {
-                A = a;
-                B = b;
-            }
-
-            public Entry A { get; }
-            public Entry B { get; }
-            public PeerConnectionState AKnowsB { get; private set; }
-            public PeerConnectionState BKnowsA { get; private set; }
-            public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
-
-            public void Check()
-            {
-                aToBTime = Measure(() => AKnowsB = Knows(A, B));
-                bToATime = Measure(() => BKnowsA = Knows(B, A));
-            }
-
-            public string GetMessage()
-            {
-                return GetResultMessage() + GetTimePostfix();
-            }
-
-            public override string ToString()
-            {
-                return $"[{GetMessage()}]";
-            }
-
-            private string GetResultMessage()
-            {
-                var aName = A.ToString();
-                var bName = B.ToString();
-
-                if (Success)
-                {
-                    return $"{aName} and {bName} know each other.";
-                }
-
-                return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}";
-            }
-
-            private string GetTimePostfix()
-            {
-                var aName = A.ToString();
-                var bName = B.ToString();
-
-                return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
-            }
-
-            private static TimeSpan Measure(Action action)
-            {
-                var start = DateTime.UtcNow;
-                action();
-                return DateTime.UtcNow - start;
-            }
-
-            private PeerConnectionState Knows(Entry a, Entry b)
-            {
-                lock (a)
-                {
-                    var peerId = b.Response.id;
-
-                    try
-                    {
-                        var response = a.Node.GetDebugPeer(peerId, timeout);
-                        if (!response.IsPeerFound)
-                        {
-                            return PeerConnectionState.NoConnection;
-                        }
-                        if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
-                        {
-                            return PeerConnectionState.Connection;
-                        }
-                    }
-                    catch
-                    {
-                    }
-
-                    // Didn't get a conclusive answer. Try again later.
-                    return PeerConnectionState.Unknown;
-                }
-            }
-        }
-    }
 }

DistTestCore/Helpers/PeerDownloadTestHelpers.cs

@@ -1,74 +1,63 @@
-using DistTestCore.Codex;
+using static DistTestCore.Helpers.FullConnectivityHelper;
-using NUnit.Framework;
 
 namespace DistTestCore.Helpers
 {
-    public class PeerDownloadTestHelpers
+    public class PeerDownloadTestHelpers : IFullConnectivityImplementation
     {
+        private readonly FullConnectivityHelper helper;
         private readonly DistTest test;
+        private ByteSize testFileSize;
 
         public PeerDownloadTestHelpers(DistTest test)
         {
+            helper = new FullConnectivityHelper(test, this);
+            testFileSize = 1.MB();
             this.test = test;
         }
 
         public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
         {
-            test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            this.testFileSize = testFileSize;
-            var start = DateTime.UtcNow;
+            helper.AssertFullyConnected(nodes);
-
-            foreach (var node in nodes)
-            {
-                var uploader = node;
-                var downloaders = nodes.Where(n => n != uploader).ToArray();
-
-                test.ScopedTestFiles(() =>
-                {
-                    PerformTest(uploader, downloaders, testFileSize);
-                });
         }
 
-            test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
+        public string Description()
-            var timeTaken = DateTime.UtcNow - start;
+        {
-
+            return "Download Connectivity";
-            AssertTimePerMB(timeTaken, nodes.Count(), testFileSize);
         }
 
-        private void AssertTimePerMB(TimeSpan timeTaken, int numberOfNodes, ByteSize size)
+        public string ValidateEntry(Entry entry, Entry[] allEntries)
         {
-            var numberOfDownloads = numberOfNodes * (numberOfNodes - 1);
+            return string.Empty;
-            var timePerDownload = timeTaken / numberOfDownloads;
-            float sizeInMB = size.ToMB();
-            var timePerMB = timePerDownload / sizeInMB;
-
-            test.Log($"Performed {numberOfDownloads} downloads of {size} in {timeTaken.TotalSeconds} seconds, for an average of {timePerMB.TotalSeconds} seconds per MB.");
-
-            Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached.");
         }
 
-        private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
+        public PeerConnectionState Check(Entry from, Entry to)
         {
-            // Generate 1 test file per downloader.
+            var expectedFile = GenerateTestFile(from.Node, to.Node);
-            var files = downloaders.Select(d => GenerateTestFile(uploader, d, testFileSize)).ToArray();
 
-            // Upload all the test files to the uploader.
+            var contentId = from.Node.UploadFile(expectedFile);
-            var contentIds = files.Select(uploader.UploadFile).ToArray();
 
-            // Each downloader should retrieve its own test file.
+            try
-            for (var i = 0; i < downloaders.Length; i++)
             {
-                var expectedFile = files[i];
+                var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded");
-                var downloadedFile = downloaders[i].DownloadContent(contentIds[i], $"{expectedFile.Label}DOWNLOADED");
-
                 expectedFile.AssertIsEqual(downloadedFile);
+                return PeerConnectionState.Connection;
             }
+            catch
+            {
+                // Should an exception occur during the download or file-content assertion,
+                // We consider that as no-connection for the purpose of this test.
+                return PeerConnectionState.NoConnection;
             }
 
-        private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader, ByteSize testFileSize)
+            // Should an exception occur during upload, then this try is inconclusive and we try again next loop.
+        }
+
+        private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader)
         {
             var up = uploader.GetName().Replace("<", "").Replace(">", "");
             var down = downloader.GetName().Replace("<", "").Replace(">", "");
-            var label = $"FROM{up}TO{down}";
+            var label = $"~from:{up}-to:{down}~";
             return test.GenerateTestFile(testFileSize, label);
         }
     }

DistTestCore/Http.cs

@@ -13,16 +13,14 @@ namespace DistTestCore
         private readonly Address address;
         private readonly string baseUrl;
         private readonly string? logAlias;
-        private readonly TimeSpan? timeoutOverride;
 
-        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null, TimeSpan? timeoutOverride = null)
+        public Http(BaseLog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
         {
             this.log = log;
             this.timeSet = timeSet;
             this.address = address;
             this.baseUrl = baseUrl;
             this.logAlias = logAlias;
-            this.timeoutOverride = timeoutOverride;
             if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
             if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
         }
@@ -127,24 +125,13 @@ namespace DistTestCore
 
         private T Retry<T>(Func<T> operation, string description)
         {
-            return Time.Retry(operation, GetTimeout(), timeSet.HttpCallRetryDelay(), description);
+            return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
         }
 
         private HttpClient GetClient()
-        {
-            return GetClient(GetTimeout());
-        }
-
-        private TimeSpan GetTimeout()
-        {
-            if (timeoutOverride.HasValue) return timeoutOverride.Value;
-            return timeSet.HttpCallTimeout();
-        }
-
-        private HttpClient GetClient(TimeSpan timeout)
         {
             var client = new HttpClient();
-            client.Timeout = timeout;
+            client.Timeout = timeSet.HttpCallTimeout();
             return client;
         }
     }

DistTestCore/OnlineCodexNode.cs

@@ -12,7 +12,6 @@ namespace DistTestCore
         string GetName();
         CodexDebugResponse GetDebugInfo();
         CodexDebugPeerResponse GetDebugPeer(string peerId);
-        CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout);
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(IOnlineCodexNode node);
@@ -60,11 +59,6 @@ namespace DistTestCore
             return CodexAccess.GetDebugPeer(peerId);
         }
 
-        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
-        {
-            return CodexAccess.GetDebugPeer(peerId, timeout);
-        }
-
         public ContentId UploadFile(TestFile file)
         {
             using var fileStream = File.OpenRead(file.Filename);
@@ -78,9 +72,6 @@ namespace DistTestCore
             if (string.IsNullOrEmpty(response)) Assert.Fail("Received empty response.");
             if (response.StartsWith(UploadFailedMessage)) Assert.Fail("Node failed to store block.");
 
-            var logReplacement = $"(CID:{file.Describe()})";
-            Log($"ContentId '{response}' is {logReplacement}");
-            lifecycle.Log.AddStringReplace(response, logReplacement);
             Log($"Uploaded file. Received contentId: '{response}'.");
             return new ContentId(response);
         }

DistTestCore/TestLifecycle.cs

@@ -25,6 +25,8 @@ namespace DistTestCore
             PrometheusStarter = new PrometheusStarter(this, workflowCreator);
             GethStarter = new GethStarter(this, workflowCreator);
             testStart = DateTime.UtcNow;
+
+            Log.WriteLogTag();
         }
 
         public BaseLog Log { get; }

DistTestCore/Timing.cs

@@ -10,7 +10,7 @@ namespace DistTestCore
     public interface ITimeSet
     {
         TimeSpan HttpCallTimeout();
-        TimeSpan HttpCallRetryTimeout();
+        TimeSpan HttpCallRetryTime();
         TimeSpan HttpCallRetryDelay();
         TimeSpan WaitForK8sServiceDelay();
         TimeSpan K8sOperationTimeout();
@@ -24,7 +24,7 @@ namespace DistTestCore
             return TimeSpan.FromSeconds(10);
         }
 
-        public TimeSpan HttpCallRetryTimeout()
+        public TimeSpan HttpCallRetryTime()
         {
             return TimeSpan.FromMinutes(1);
         }
@@ -57,7 +57,7 @@ namespace DistTestCore
             return TimeSpan.FromHours(2);
         }
 
-        public TimeSpan HttpCallRetryTimeout()
+        public TimeSpan HttpCallRetryTime()
         {
             return TimeSpan.FromHours(5);
         }

Logging/BaseLog.cs

@@ -73,6 +73,14 @@ namespace Logging
             return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
         }
 
+        public void WriteLogTag()
+        {
+            var runId = NameUtils.GetRunId();
+            var category = NameUtils.GetCategoryName();
+            var name = NameUtils.GetTestMethodName();
+            LogFile.WriteRaw($"{runId} {category} {name}");
+        }
+
         private string ApplyReplacements(string str)
         {
             foreach (var replacement in replacements)

Logging/FixtureLog.cs

@@ -1,20 +1,14 @@
-using NUnit.Framework;
+namespace Logging
-
-namespace Logging
 {
     public class FixtureLog : BaseLog
     {
-        private readonly DateTime start;
         private readonly string fullName;
         private readonly LogConfig config;
 
-        public FixtureLog(LogConfig config, string name = "")
+        public FixtureLog(LogConfig config, DateTime start, string name = "")
             : base(config.DebugEnabled)
         {
-            start = DateTime.UtcNow;
+            fullName = NameUtils.GetFixtureFullName(config, start, name);
-            var folder = DetermineFolder(config);
-            var fixtureName = GetFixtureName(name);
-            fullName = Path.Combine(folder, fixtureName);
             this.config = config;
         }
 
@@ -32,28 +26,5 @@ namespace Logging
         {
             return fullName;
         }
-
-        private string DetermineFolder(LogConfig config)
-        {
-            return Path.Join(
-               config.LogRoot,
-               $"{start.Year}-{Pad(start.Month)}",
-               Pad(start.Day));
-        }
-
-        private string GetFixtureName(string name)
-        {
-            var test = TestContext.CurrentContext.Test;
-            var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
-            if (!string.IsNullOrEmpty(name)) className = name;
-
-            return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
-        }
-
-        private static string Pad(int n)
-        {
-            return n.ToString().PadLeft(2, '0');
-        }
-
     }
 }

Logging/LogFile.cs

@@ -49,7 +49,7 @@
 
         private static string GetTimestamp()
         {
-            return $"[{DateTime.UtcNow.ToString("u")}]";
+            return $"[{DateTime.UtcNow.ToString("o")}]";
         }
 
         private void EnsurePathExists(string filename)

Logging/NameUtils.cs

@@ -0,0 +1,83 @@
+using NUnit.Framework;
+
+namespace Logging
+{
+    public static class NameUtils
+    {
+        public static string GetTestMethodName(string name = "")
+        {
+            if (!string.IsNullOrEmpty(name)) return name;
+            var test = TestContext.CurrentContext.Test;
+            var args = FormatArguments(test);
+            return ReplaceInvalidCharacters($"{test.MethodName}{args}");
+        }
+
+        public static string GetFixtureFullName(LogConfig config, DateTime start, string name)
+        {
+            var folder = DetermineFolder(config, start);
+            var fixtureName = GetFixtureName(name, start);
+            return Path.Combine(folder, fixtureName);
+        }
+
+        public static string GetRawFixtureName()
+        {
+            var test = TestContext.CurrentContext.Test;
+            var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
+            return className.Replace('.', '-');
+        }
+
+        public static string GetCategoryName()
+        {
+            var test = TestContext.CurrentContext.Test;
+            return test.ClassName!.Substring(0, test.ClassName.LastIndexOf('.'));
+        }
+
+        public static string GetTestId()
+        {
+            return GetEnvVar("TESTID");
+        }
+
+        public static string GetRunId()
+        {
+            return GetEnvVar("RUNID");
+        }
+
+        private static string GetEnvVar(string name)
+        {
+            var v = Environment.GetEnvironmentVariable(name);
+            if (string.IsNullOrEmpty(v)) return $"EnvVar'{name}'NotSet";
+            return v;
+        }
+
+        private static string FormatArguments(TestContext.TestAdapter test)
+        {
+            if (test.Arguments == null || !test.Arguments.Any()) return "";
+            return $"[{string.Join(',', test.Arguments)}]";
+        }
+
+        private static string ReplaceInvalidCharacters(string name)
+        {
+            return name.Replace(":", "_");
+        }
+
+        private static string DetermineFolder(LogConfig config, DateTime start)
+        {
+            return Path.Join(
+               config.LogRoot,
+               $"{start.Year}-{Pad(start.Month)}",
+               Pad(start.Day));
+        }
+
+        private static string GetFixtureName(string name, DateTime start)
+        {
+            var className = GetRawFixtureName();
+            if (!string.IsNullOrEmpty(name)) className = name;
+            return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
+        }
+
+        private static string Pad(int n)
+        {
+            return n.ToString().PadLeft(2, '0');
+        }
+    }
+}

Logging/StatusLog.cs

@@ -0,0 +1,63 @@
+using Newtonsoft.Json;
+
+namespace Logging
+{
+    public class StatusLog
+    {
+        private readonly object fileLock = new object();
+        private readonly string fullName;
+        private readonly string fixtureName;
+        private readonly string codexId;
+
+        public StatusLog(LogConfig config, DateTime start, string codexId, string name = "")
+        {
+            fullName = NameUtils.GetFixtureFullName(config, start, name) + "_STATUS.log";
+            fixtureName = NameUtils.GetRawFixtureName();
+            this.codexId = codexId;
+        }
+
+        public void ConcludeTest(string resultStatus, string testDuration)
+        {
+            Write(new StatusLogJson
+            {
+                @timestamp = DateTime.UtcNow.ToString("o"),
+                runid = NameUtils.GetRunId(),
+                status = resultStatus,
+                testid = NameUtils.GetTestId(),
+                codexid = codexId,
+                category = NameUtils.GetCategoryName(),
+                fixturename = fixtureName,
+                testname = NameUtils.GetTestMethodName(),
+                testduration = testDuration
+            });
+        }
+
+        private void Write(StatusLogJson json)
+        {
+            try
+            {
+                lock (fileLock)
+                {
+                    File.AppendAllLines(fullName, new[] { JsonConvert.SerializeObject(json) });
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine("Unable to write to status log: " + ex);
+            }
+        }
+    }
+
+    public class StatusLogJson
+    {
+        public string @timestamp { get; set; } = string.Empty;
+        public string runid { get; set; } = string.Empty;
+        public string status { get; set; } = string.Empty;
+        public string testid { get; set; } = string.Empty; 
+        public string codexid { get; set; } = string.Empty;
+        public string category { get; set; } = string.Empty;
+        public string fixturename { get; set; } = string.Empty;
+        public string testname { get; set; } = string.Empty;
+        public string testduration { get; set;} = string.Empty;
+    }
+}

Logging/TestLog.cs

@@ -10,7 +10,7 @@ namespace Logging
         public TestLog(string folder, bool debug, string name = "")
             : base(debug)
         {
-            methodName = GetMethodName(name);
+            methodName = NameUtils.GetTestMethodName(name);
             fullName = Path.Combine(folder, methodName);
 
             Log($"*** Begin: {methodName}");
@@ -37,24 +37,5 @@ namespace Logging
         {
             return fullName;
         }
-
-        private string GetMethodName(string name)
-        {
-            if (!string.IsNullOrEmpty(name)) return name;
-            var test = TestContext.CurrentContext.Test;
-            var args = FormatArguments(test);
-            return ReplaceInvalidCharacters($"{test.MethodName}{args}");
-        }
-
-        private static string FormatArguments(TestContext.TestAdapter test)
-        {
-            if (test.Arguments == null || !test.Arguments.Any()) return "";
-            return $"[{string.Join(',', test.Arguments)}]";
-        }
-
-        private static string ReplaceInvalidCharacters(string name)
-        {
-            return name.Replace(":", "_");
-        }
     }
 }

LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs

@@ -0,0 +1,22 @@
+using DistTestCore.Helpers;
+using DistTestCore;
+using NUnit.Framework;
+
+namespace TestsLong.DownloadConnectivityTests
+{
+    [TestFixture]
+    public class LongFullyConnectedDownloadTests : AutoBootstrapDistTest
+    {
+        [Test]
+        [UseLongTimeouts]
+        [Combinatorial]
+        public void FullyConnectedDownloadTest(
+            [Values(10, 15, 20)] int numberOfNodes,
+            [Values(10, 100)] int sizeMBs)
+        {
+            for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
+
+            PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB());
+        }
+    }
+}

Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -9,8 +9,8 @@ namespace Tests.DownloadConnectivityTests
         [Test]
         [Combinatorial]
         public void FullyConnectedDownloadTest(
-            [Values(3, 10, 20)] int numberOfNodes,
+            [Values(1, 3, 5)] int numberOfNodes,
-            [Values(1, 10, 100)] int sizeMBs)
+            [Values(1, 10)] int sizeMBs)
         {
             for (var i = 0; i < numberOfNodes; i++) SetupCodexNode();
 

docker/job.yaml

@@ -28,6 +28,10 @@ spec:
           value: ${BRANCH}
         - name: SOURCE
           value: ${SOURCE}
+        - name: RUNID
+          value: ${RUNID}
+        - name: TESTID
+          value: ${TESTID}
         volumeMounts:
         - name: kubeconfig
           mountPath: /opt/kubeconfig.yaml