From 31e034ab6776f285ed70cab65e49193602830e0c Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 13 Apr 2023 11:53:54 +0200 Subject: [PATCH] Restores automatic log download on test failure --- CodexDistTestCore/PodLogDownloader.cs | 9 ---- .../CodexNodeLog.cs | 2 +- ...ownloadLogsAndMetricsOnFailureAttribute.cs | 15 ++++++ .../LogDownloadHandler.cs | 2 +- DistTestCore/CodexNodeGroup.cs | 7 ++- DistTestCore/CodexStarter.cs | 26 +++++++-- DistTestCore/DistTest.cs | 54 +++++++++---------- DistTestCore/OnlineCodexNode.cs | 2 +- DistTestCore/TestLifecycle.cs | 3 +- 9 files changed, 71 insertions(+), 49 deletions(-) rename DistTestCore/{CodexLogs => CodexLogsAndMetrics}/CodexNodeLog.cs (95%) create mode 100644 DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs rename DistTestCore/{CodexLogs => CodexLogsAndMetrics}/LogDownloadHandler.cs (95%) diff --git a/CodexDistTestCore/PodLogDownloader.cs b/CodexDistTestCore/PodLogDownloader.cs index 4df84d2f..e09a57ce 100644 --- a/CodexDistTestCore/PodLogDownloader.cs +++ b/CodexDistTestCore/PodLogDownloader.cs @@ -7,15 +7,6 @@ namespace CodexDistTestCore void Log(Stream log); } - [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] - public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute - { - public DontDownloadLogsAndMetricsOnFailureAttribute() - : base(Timing.UseLongTimeoutsKey) - { - } - } - public class PodLogDownloader { public const string DontDownloadLogsOnFailureKey = "DontDownloadLogsOnFailure"; diff --git a/DistTestCore/CodexLogs/CodexNodeLog.cs b/DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs similarity index 95% rename from DistTestCore/CodexLogs/CodexNodeLog.cs rename to DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs index 5003bf8b..a4a9cb07 100644 --- a/DistTestCore/CodexLogs/CodexNodeLog.cs +++ b/DistTestCore/CodexLogsAndMetrics/CodexNodeLog.cs @@ -1,7 +1,7 @@ using Logging; using NUnit.Framework; -namespace DistTestCore.CodexLogs +namespace DistTestCore.CodexLogsAndMetrics { public interface ICodexNodeLog { diff --git a/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs b/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs new file mode 100644 index 00000000..0cf1cbe9 --- /dev/null +++ b/DistTestCore/CodexLogsAndMetrics/DontDownloadLogsAndMetricsOnFailureAttribute.cs @@ -0,0 +1,15 @@ +using NUnit.Framework; + +namespace DistTestCore.CodexLogsAndMetrics +{ + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute + { + public const string DontDownloadKey = "DontDownloadLogsAndMetrics"; + + public DontDownloadLogsAndMetricsOnFailureAttribute() + : base(DontDownloadKey) + { + } + } +} diff --git a/DistTestCore/CodexLogs/LogDownloadHandler.cs b/DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs similarity index 95% rename from DistTestCore/CodexLogs/LogDownloadHandler.cs rename to DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs index 3848b288..59c14d87 100644 --- a/DistTestCore/CodexLogs/LogDownloadHandler.cs +++ b/DistTestCore/CodexLogsAndMetrics/LogDownloadHandler.cs @@ -1,7 +1,7 @@ using KubernetesWorkflow; using Logging; -namespace DistTestCore.CodexLogs +namespace DistTestCore.CodexLogsAndMetrics { public class LogDownloadHandler : ILogHandler { diff --git a/DistTestCore/CodexNodeGroup.cs b/DistTestCore/CodexNodeGroup.cs index e4861629..2c5c3127 100644 --- a/DistTestCore/CodexNodeGroup.cs +++ b/DistTestCore/CodexNodeGroup.cs @@ -32,15 +32,14 @@ namespace DistTestCore public ICodexSetup BringOffline() { - var result = Setup; - var containers = Containers; + lifecycle.CodexStarter.BringOffline(this); + var result = Setup; // Clear everything. Prevent accidental use. Setup = null!; - Containers = null!; Nodes = Array.Empty(); + Containers = null!; - lifecycle.CodexStarter.BringOffline(containers); return result; } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index 41c685fa..d7bc0ab2 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,7 +1,5 @@ using DistTestCore.Codex; -using DistTestCore.CodexLogs; using KubernetesWorkflow; -using Nethereum.Merkle.Patricia; namespace DistTestCore { @@ -16,27 +14,40 @@ namespace DistTestCore this.lifecycle = lifecycle; } + public List RunningGroups { get; } = new List(); + public ICodexNodeGroup BringOnline(CodexSetup codexSetup) { + Log($"Starting {codexSetup.Describe()}..."); + var workflow = CreateWorkflow(); var startupConfig = new StartupConfig(); startupConfig.Add(codexSetup); var runningContainers = workflow.Start(codexSetup.NumberOfNodes, codexSetup.Location, new CodexContainerRecipe(), startupConfig); - return new CodexNodeGroup(lifecycle, codexSetup, runningContainers); + var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers); + RunningGroups.Add(group); + + Log($"Started at '{group.Containers.RunningPod.Ip}'"); + return group; } - public void BringOffline(RunningContainers runningContainers) + public void BringOffline(CodexNodeGroup group) { + Log($"Stopping {group.Describe()}..."); var workflow = CreateWorkflow(); - workflow.Stop(runningContainers); + workflow.Stop(group.Containers); + RunningGroups.Remove(group); + Log("Stopped."); } public void DeleteAllResources() { var workflow = CreateWorkflow(); workflow.DeleteAllResources(); + + RunningGroups.Clear(); } public void DownloadLog(RunningContainer container, ILogHandler logHandler) @@ -49,5 +60,10 @@ namespace DistTestCore { return workflowCreator.CreateWorkflow(); } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } } } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 9a43a7b3..d1830baf 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -1,4 +1,5 @@ -using NUnit.Framework; +using DistTestCore.CodexLogsAndMetrics; +using NUnit.Framework; namespace DistTestCore { @@ -68,20 +69,20 @@ namespace DistTestCore private void IncludeLogsAndMetricsOnTestFailure() { - //var result = TestContext.CurrentContext.Result; - //if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - //{ - // if (IsDownloadingLogsAndMetricsEnabled()) - // { - // log.Log("Downloading all CodexNode logs and metrics because of test failure..."); - // k8sManager.ForEachOnlineGroup(DownloadLogs); - // k8sManager.DownloadAllMetrics(); - // } - // else - // { - // log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); - // } - //} + var result = TestContext.CurrentContext.Result; + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + if (IsDownloadingLogsAndMetricsEnabled()) + { + Log("Downloading all CodexNode logs and metrics because of test failure..."); + DownloadAllLogs(); + //k8sManager.DownloadAllMetrics(); + } + else + { + Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); + } + } } private void Log(string msg) @@ -99,21 +100,20 @@ namespace DistTestCore lifecycle = new TestLifecycle(new Configuration()); } - private void DownloadLogs(CodexNodeGroup group) + private void DownloadAllLogs() { - //foreach (var node in group) - //{ - // var downloader = new PodLogDownloader(log, k8sManager); - // var n = (OnlineCodexNode)node; - // downloader.DownloadLog(n); - //} + var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes); + foreach (var node in allNodes) + { + lifecycle.DownloadLog(node); + } } - //private bool IsDownloadingLogsAndMetricsEnabled() - //{ - // var testProperties = TestContext.CurrentContext.Test.Properties; - // return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); - //} + private bool IsDownloadingLogsAndMetricsEnabled() + { + var testProperties = TestContext.CurrentContext.Test.Properties; + return !testProperties.ContainsKey(DontDownloadLogsAndMetricsOnFailureAttribute.DontDownloadKey); + } } public static class GlobalTestFailure diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 9ecaef5e..8f85f91d 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -1,5 +1,5 @@ using DistTestCore.Codex; -using DistTestCore.CodexLogs; +using DistTestCore.CodexLogsAndMetrics; using NUnit.Framework; namespace DistTestCore diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 95f4508e..12672165 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,4 +1,4 @@ -using DistTestCore.CodexLogs; +using DistTestCore.CodexLogsAndMetrics; using Logging; namespace DistTestCore @@ -28,6 +28,7 @@ namespace DistTestCore var description = node.Describe(); var handler = new LogDownloadHandler(description, subFile); + Log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}"); CodexStarter.DownloadLog(node.CodexAccess.Container, handler); return new CodexNodeLog(subFile);