From 8caa7ab4fa95496a4f05ef54e716ef29b51a0978 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 27 Sep 2023 11:33:54 +0200 Subject: [PATCH] Adds option to limit continuous test run to set duration. --- Tests/CodexContinuousTests/Configuration.cs | 3 ++ .../ContinuousTestRunner.cs | 17 +++++++++- Tests/CodexContinuousTests/SingleTestRun.cs | 2 +- .../Tests/TwoClientTest.cs | 33 ++++++++----------- Tests/CodexContinuousTests/run.sh | 3 +- 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/Tests/CodexContinuousTests/Configuration.cs b/Tests/CodexContinuousTests/Configuration.cs index 50ef578..e84a253 100644 --- a/Tests/CodexContinuousTests/Configuration.cs +++ b/Tests/CodexContinuousTests/Configuration.cs @@ -24,6 +24,9 @@ namespace ContinuousTests [Uniform("stop", "s", "STOPONFAIL", false, "If greater than zero, runner will stop after this many test failures and download all cluster container logs. 0 by default.")] public int StopOnFailure { get; set; } = 0; + [Uniform("target-duration", "td", "TARGETDURATION", false, "If greater than zero, runner will run for this many seconds before stopping.")] + public int TargetDurationSeconds { get; set; } = 0; + [Uniform("dl-logs", "dl", "DLLOGS", false, "If true, runner will periodically download and save/append container logs to the log path.")] public bool DownloadContainerLogs { get; set; } = false; diff --git a/Tests/CodexContinuousTests/ContinuousTestRunner.cs b/Tests/CodexContinuousTests/ContinuousTestRunner.cs index a0589b2..90c0d20 100644 --- a/Tests/CodexContinuousTests/ContinuousTestRunner.cs +++ b/Tests/CodexContinuousTests/ContinuousTestRunner.cs @@ -1,5 +1,6 @@ using DistTestCore.Logs; using Logging; +using Utils; namespace ContinuousTests { @@ -53,12 +54,26 @@ namespace ContinuousTests } overviewLog.Log("Finished launching test-loops."); - cancelToken.WaitHandle.WaitOne(); + WaitUntilFinished(overviewLog); overviewLog.Log("Cancelling all test-loops..."); taskFactory.WaitAll(); overviewLog.Log("All tasks cancelled."); } + private void WaitUntilFinished(LogSplitter overviewLog) + { + if (config.TargetDurationSeconds > 0) + { + var targetDuration = TimeSpan.FromSeconds(config.TargetDurationSeconds); + cancelToken.WaitHandle.WaitOne(targetDuration); + overviewLog.Log($"Congratulations! The targer duration has been reached! ({Time.FormatDuration(targetDuration)})"); + } + else + { + cancelToken.WaitHandle.WaitOne(); + } + } + private void ClearAllCustomNamespaces(ContinuousTest[] allTests, ILog log) { foreach (var test in allTests) ClearAllCustomNamespaces(test, log); diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/CodexContinuousTests/SingleTestRun.cs index 7a233db..7c3b30f 100644 --- a/Tests/CodexContinuousTests/SingleTestRun.cs +++ b/Tests/CodexContinuousTests/SingleTestRun.cs @@ -31,7 +31,7 @@ namespace ContinuousTests this.handle = handle; this.cancelToken = cancelToken; testName = handle.Test.GetType().Name; - fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName); + fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), DateTime.UtcNow, testName); entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, fixtureLog); ApplyLogReplacements(fixtureLog, startupChecker); diff --git a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs index bf08bae..6a382be 100644 --- a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs +++ b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs @@ -23,11 +23,10 @@ namespace ContinuousTests.Tests { file = FileManager.GenerateFile(size); - AssertBytesStoredMetric(Nodes[0], () => - { - LogBytesPerMillisecond(() => cid = Nodes[0].UploadFile(file)); - Assert.That(cid, Is.Not.Null); - }); + LogStoredBytes(Nodes[0]); + + LogBytesPerMillisecond(() => cid = Nodes[0].UploadFile(file)); + Assert.That(cid, Is.Not.Null); } [TestMoment(t: 10)] @@ -40,25 +39,19 @@ namespace ContinuousTests.Tests file.AssertIsEqual(dl); } - private void AssertBytesStoredMetric(ICodexNode node, Action action) + private void LogStoredBytes(ICodexNode node) { - var lowExpected = size.SizeInBytes; - var highExpected = size.SizeInBytes * 1.2; - var metrics = CreateMetricsAccess(node); - var before = metrics.GetMetric(BytesStoredMetric); - - action(); - - Log.Log($"Waiting for between {lowExpected} and {highExpected} new bytes to be stored by node {node.GetName()}."); - - Time.WaitUntil(() => + var metric = metrics.GetMetric(BytesStoredMetric); + if (metric == null) { - var after = metrics.GetMetric(BytesStoredMetric); - var newBytes = Convert.ToInt64(after.Values.Last().Value - before.Values.Last().Value); + Log.Log($"Unabled to fetch metric '{BytesStoredMetric}' for node '{node.GetName()}'"); + return; + } - return highExpected > newBytes && newBytes > lowExpected; - }); + var bytes = new ByteSize(Convert.ToInt64(metric.Values.Single().Value)); + + Log.Log($"{node.GetName()}: {bytes}"); } private void LogBytesPerMillisecond(Action action) diff --git a/Tests/CodexContinuousTests/run.sh b/Tests/CodexContinuousTests/run.sh index 6b281d7..72822a8 100644 --- a/Tests/CodexContinuousTests/run.sh +++ b/Tests/CodexContinuousTests/run.sh @@ -3,4 +3,5 @@ dotnet run \ --codex-deployment=codex-deployment.json \ --keep=1 \ --stop=10 \ - --dl-logs=1 + --dl-logs=1 \ + --target-duration=172800 # 48 hours