Adds option to limit continuous test run to set duration.

Tests/CodexContinuousTests/Configuration.cs

@@ -24,6 +24,9 @@ namespace ContinuousTests
         [Uniform("stop", "s", "STOPONFAIL", false, "If greater than zero, runner will stop after this many test failures and download all cluster container logs. 0 by default.")]
         public int StopOnFailure { get; set; } = 0;
 
+        [Uniform("target-duration", "td", "TARGETDURATION", false, "If greater than zero, runner will run for this many seconds before stopping.")]
+        public int TargetDurationSeconds { get; set; } = 0;
+
         [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")]
         public bool DownloadContainerLogs { get; set; } = false;
 

Tests/CodexContinuousTests/ContinuousTestRunner.cs

@@ -1,5 +1,6 @@
 using DistTestCore.Logs;
 using Logging;
+using Utils;
 
 namespace ContinuousTests
 {
@@ -53,12 +54,26 @@ namespace ContinuousTests
             }
 
             overviewLog.Log("Finished launching test-loops.");
-            cancelToken.WaitHandle.WaitOne();
+            WaitUntilFinished(overviewLog);
             overviewLog.Log("Cancelling all test-loops...");
             taskFactory.WaitAll();
             overviewLog.Log("All tasks cancelled.");
         }
 
+        private void WaitUntilFinished(LogSplitter overviewLog)
+        {
+            if (config.TargetDurationSeconds > 0)
+            {
+                var targetDuration = TimeSpan.FromSeconds(config.TargetDurationSeconds);
+                cancelToken.WaitHandle.WaitOne(targetDuration);
+                overviewLog.Log($"Congratulations! The targer duration has been reached! ({Time.FormatDuration(targetDuration)})");
+            }
+            else
+            {
+                cancelToken.WaitHandle.WaitOne();
+            }
+        }
+
         private void ClearAllCustomNamespaces(ContinuousTest[] allTests, ILog log)
         {
             foreach (var test in allTests) ClearAllCustomNamespaces(test, log);

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -31,7 +31,7 @@ namespace ContinuousTests
             this.handle = handle;
             this.cancelToken = cancelToken;
             testName = handle.Test.GetType().Name;
-            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
+            fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, testName);
             entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, fixtureLog);
             ApplyLogReplacements(fixtureLog, startupChecker);
 

Tests/CodexContinuousTests/Tests/TwoClientTest.cs

@@ -23,11 +23,10 @@ namespace ContinuousTests.Tests
         {
             file = FileManager.GenerateFile(size);
 
-            AssertBytesStoredMetric(Nodes[0], () =>
-            {
-                LogBytesPerMillisecond(() => cid = Nodes[0].UploadFile(file));
-                Assert.That(cid, Is.Not.Null);
-            });
+            LogStoredBytes(Nodes[0]);
+
+            LogBytesPerMillisecond(() => cid = Nodes[0].UploadFile(file));
+            Assert.That(cid, Is.Not.Null);
         }
 
         [TestMoment(t: 10)]
@@ -40,25 +39,19 @@ namespace ContinuousTests.Tests
             file.AssertIsEqual(dl);
         }
 
-        private void AssertBytesStoredMetric(ICodexNode node, Action action)
+        private void LogStoredBytes(ICodexNode node)
         {
-            var lowExpected = size.SizeInBytes;
-            var highExpected = size.SizeInBytes * 1.2;
-
             var metrics = CreateMetricsAccess(node);
-            var before = metrics.GetMetric(BytesStoredMetric);
-
-            action();
-
-            Log.Log($"Waiting for between {lowExpected} and {highExpected} new bytes to be stored by node {node.GetName()}.");
-
-            Time.WaitUntil(() =>
+            var metric = metrics.GetMetric(BytesStoredMetric);
+            if (metric == null)
             {
-                var after = metrics.GetMetric(BytesStoredMetric);
-                var newBytes = Convert.ToInt64(after.Values.Last().Value - before.Values.Last().Value);
+                Log.Log($"Unabled to fetch metric '{BytesStoredMetric}' for node '{node.GetName()}'");
+                return;
+            }
 
-                return highExpected > newBytes && newBytes > lowExpected;
-            });
+            var bytes = new ByteSize(Convert.ToInt64(metric.Values.Single().Value));
+
+            Log.Log($"{node.GetName()}: {bytes}");
         }
 
         private void LogBytesPerMillisecond(Action action)

Tests/CodexContinuousTests/run.sh

@@ -3,4 +3,5 @@ dotnet run \
     --codex-deployment=codex-deployment.json \
     --keep=1 \
     --stop=10 \
-    --dl-logs=1
+    --dl-logs=1 \
+    --target-duration=172800 # 48 hours