From 015d8da21d8487e7eeabdff124458edcc919cb7c Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 14 Apr 2024 09:17:25 +0200 Subject: [PATCH] Better logging for Time.WaitUntil. --- Framework/KubernetesWorkflow/K8sController.cs | 2 +- Framework/Utils/Time.cs | 13 ++++++++----- .../CodexContractsPlugin/CodexContractsStarter.cs | 8 ++++---- Tests/CodexTests/BasicTests/ContinuousSubstitute.cs | 2 +- Tests/CodexTests/BasicTests/NetworkIsolationTest.cs | 4 ++-- Tests/DistTestCore/Helpers/AssertHelpers.cs | 2 +- 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/Framework/KubernetesWorkflow/K8sController.cs b/Framework/KubernetesWorkflow/K8sController.cs index d81b87c..71430a7 100644 --- a/Framework/KubernetesWorkflow/K8sController.cs +++ b/Framework/KubernetesWorkflow/K8sController.cs @@ -904,7 +904,7 @@ namespace KubernetesWorkflow var sw = Stopwatch.Begin(log, true); try { - Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay()); + Time.WaitUntil(predicate, cluster.K8sOperationTimeout(), cluster.K8sOperationRetryDelay(), msg); } finally { diff --git a/Framework/Utils/Time.cs b/Framework/Utils/Time.cs index 82a836e..e54ed18 100644 --- a/Framework/Utils/Time.cs +++ b/Framework/Utils/Time.cs @@ -57,24 +57,27 @@ return result; } - public static void WaitUntil(Func predicate) + public static void WaitUntil(Func predicate, string msg) { - WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1)); + WaitUntil(predicate, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(1), msg); } - public static void WaitUntil(Func predicate, TimeSpan timeout, TimeSpan retryDelay) + public static void WaitUntil(Func predicate, TimeSpan timeout, TimeSpan retryDelay, string msg) { var start = DateTime.UtcNow; + var tries = 1; var state = predicate(); while (!state) { - if (DateTime.UtcNow - start > timeout) + var duration = DateTime.UtcNow - start; + if (duration > timeout) { - throw new TimeoutException("Operation timed out."); + throw new TimeoutException($"Operation timed out after {tries} tries over (total) {FormatDuration(duration)}. '{msg}'"); } Sleep(retryDelay); state = predicate(); + tries++; } } diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs index 0137a17..d2b5c3f 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs @@ -59,7 +59,7 @@ namespace CodexContractsPlugin var logHandler = new ContractsReadyLogHandler(tools.GetLog()); workflow.DownloadContainerLog(container, logHandler, 100); return logHandler.Found; - }); + }, nameof(DeployContract)); Log("Contracts deployed. Extracting addresses..."); var extractor = new ContractsContainerInfoExtractor(tools.GetLog(), workflow, container); @@ -71,7 +71,7 @@ namespace CodexContractsPlugin Log("Extract completed. Checking sync..."); - Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi)); + Time.WaitUntil(() => interaction.IsSynced(marketplaceAddress, abi), nameof(DeployContract)); Log("Synced. Codex SmartContracts deployed."); @@ -83,9 +83,9 @@ namespace CodexContractsPlugin tools.GetLog().Log(msg); } - private void WaitUntil(Func predicate) + private void WaitUntil(Func predicate, string msg) { - Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2)); + Time.WaitUntil(predicate, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(2), msg); } private StartupConfig CreateStartupConfig(IGethNode gethNode) diff --git a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs index 2898be2..e135d2f 100644 --- a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs @@ -154,7 +154,7 @@ namespace CodexTests.BasicTests var newBytes = Convert.ToInt64(afterBytesStored.Values.Last().Value - beforeBytesStored.Values.Last().Value); return high > newBytes && newBytes > low; - }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2)); + }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(2), nameof(ContinuousSubstitute)); FileUtils.TrackedFile? downloadedFile = null; LogBytesPerMillisecond(() => downloadedFile = secondary.DownloadContent(contentId)); diff --git a/Tests/CodexTests/BasicTests/NetworkIsolationTest.cs b/Tests/CodexTests/BasicTests/NetworkIsolationTest.cs index ccacdba..2604550 100644 --- a/Tests/CodexTests/BasicTests/NetworkIsolationTest.cs +++ b/Tests/CodexTests/BasicTests/NetworkIsolationTest.cs @@ -19,7 +19,7 @@ namespace CodexTests.BasicTests { node = Ci.StartCodexNode(); - Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5)); + Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5), nameof(SetUpANodeAndWait)); } [Test] @@ -27,7 +27,7 @@ namespace CodexTests.BasicTests { var myNode = Ci.StartCodexNode(); - Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5)); + Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5), nameof(ForeignNodeConnects)); try { diff --git a/Tests/DistTestCore/Helpers/AssertHelpers.cs b/Tests/DistTestCore/Helpers/AssertHelpers.cs index efd0749..c0f995d 100644 --- a/Tests/DistTestCore/Helpers/AssertHelpers.cs +++ b/Tests/DistTestCore/Helpers/AssertHelpers.cs @@ -14,7 +14,7 @@ namespace DistTestCore.Helpers Time.WaitUntil(() => { var c = constraint.Resolve(); return c.ApplyTo(actual()).IsSuccess; - }); + }, "RetryAssert: " + message); } catch (TimeoutException) {