From a2e07fbd2e50a86133a84234b58aa54422bb2e63 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 13 Sep 2023 15:10:19 +0200 Subject: [PATCH] Automatic downloading of container logs on test failure --- DistTestCore/Configuration.cs | 7 ++++ DistTestCore/DistTest.cs | 17 +++++---- ...ownloadLogsAndMetricsOnFailureAttribute.cs | 15 -------- .../DontDownloadLogsOnFailureAttribute.cs | 15 ++++++++ DistTestCore/TestLifecycle.cs | 36 +++++++++++++++++-- KubernetesWorkflow/Configuration.cs | 1 + KubernetesWorkflow/K8sHooks.cs | 19 ++++++++++ KubernetesWorkflow/StartupWorkflow.cs | 5 ++- 8 files changed, 88 insertions(+), 27 deletions(-) delete mode 100644 DistTestCore/DontDownloadLogsAndMetricsOnFailureAttribute.cs create mode 100644 DistTestCore/DontDownloadLogsOnFailureAttribute.cs create mode 100644 KubernetesWorkflow/K8sHooks.cs diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index b22415d..35b9823 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -1,4 +1,5 @@ using Core; +using KubernetesWorkflow; namespace DistTestCore { @@ -26,6 +27,11 @@ namespace DistTestCore } public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace) + { + return GetK8sConfiguration(timeSet, new DoNothingK8sHooks(), k8sNamespace); + } + + public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, IK8sHooks hooks, string k8sNamespace) { var config = new KubernetesWorkflow.Configuration( kubeConfigFile: kubeConfigFile, @@ -35,6 +41,7 @@ namespace DistTestCore ); config.AllowNamespaceOverride = false; + config.Hooks = hooks; return config; } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 5ae23a4..408576c 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -171,7 +171,7 @@ namespace DistTestCore { WriteEndTestLog(lifecycle.Log); - IncludeLogsAndMetricsOnTestFailure(lifecycle); + IncludeLogsOnTestFailure(lifecycle); lifecycle.DeleteAllResources(); lifecycle = null!; }); @@ -215,22 +215,21 @@ namespace DistTestCore return testMethods.Any(m => m.GetCustomAttribute() != null); } - private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle) + private void IncludeLogsOnTestFailure(TestLifecycle lifecycle) { var result = TestContext.CurrentContext.Result; if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) { fixtureLog.MarkAsFailed(); - if (IsDownloadingLogsAndMetricsEnabled()) + if (IsDownloadingLogsEnabled()) { - lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure..."); - //DownloadAllLogs(lifecycle); - //DownloadAllMetrics(lifecycle); + lifecycle.Log.Log("Downloading all container logs because of test failure..."); + lifecycle.DownloadAllLogs(); } else { - lifecycle.Log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); + lifecycle.Log.Log("Skipping download of all container logs due to [DontDownloadLogsOnFailure] attribute."); } } } @@ -245,10 +244,10 @@ namespace DistTestCore return TestContext.CurrentContext.Result.Outcome.Status.ToString(); } - private bool IsDownloadingLogsAndMetricsEnabled() + private bool IsDownloadingLogsEnabled() { var testProperties = TestContext.CurrentContext.Test.Properties; - return !testProperties.ContainsKey(DontDownloadLogsAndMetricsOnFailureAttribute.DontDownloadKey); + return !testProperties.ContainsKey(DontDownloadLogsOnFailureAttribute.DontDownloadKey); } } diff --git a/DistTestCore/DontDownloadLogsAndMetricsOnFailureAttribute.cs b/DistTestCore/DontDownloadLogsAndMetricsOnFailureAttribute.cs deleted file mode 100644 index 335bbe3..0000000 --- a/DistTestCore/DontDownloadLogsAndMetricsOnFailureAttribute.cs +++ /dev/null @@ -1,15 +0,0 @@ -using NUnit.Framework; - -namespace DistTestCore -{ - [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] - public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute - { - public const string DontDownloadKey = "DontDownloadLogsAndMetrics"; - - public DontDownloadLogsAndMetricsOnFailureAttribute() - : base(DontDownloadKey) - { - } - } -} diff --git a/DistTestCore/DontDownloadLogsOnFailureAttribute.cs b/DistTestCore/DontDownloadLogsOnFailureAttribute.cs new file mode 100644 index 0000000..800b35b --- /dev/null +++ b/DistTestCore/DontDownloadLogsOnFailureAttribute.cs @@ -0,0 +1,15 @@ +using NUnit.Framework; + +namespace DistTestCore +{ + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public class DontDownloadLogsOnFailureAttribute : PropertyAttribute + { + public const string DontDownloadKey = "DontDownloadLogs"; + + public DontDownloadLogsOnFailureAttribute() + : base(DontDownloadKey) + { + } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index ef04186..999beb7 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,14 +1,16 @@ using Core; using FileUtils; +using KubernetesWorkflow; using Logging; using Utils; namespace DistTestCore { - public class TestLifecycle + public class TestLifecycle : IK8sHooks { private readonly DateTime testStart; private readonly EntryPoint entryPoint; + private readonly List runningContainers = new List(); public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace) { @@ -17,7 +19,7 @@ namespace DistTestCore TimeSet = timeSet; testStart = DateTime.UtcNow; - entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, testNamespace), configuration.GetFileManagerFolder(), timeSet); + entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet); CoreInterface = entryPoint.CreateInterface(); log.WriteLogTag(); @@ -51,6 +53,36 @@ namespace DistTestCore return Time.FormatDuration(testDuration); } + public void OnContainersStarted(RunningContainers rc) + { + runningContainers.Add(rc); + } + + public void OnContainersStopped(RunningContainers rc) + { + runningContainers.Remove(rc); + } + + public void DownloadAllLogs() + { + var workflow = entryPoint.Tools.CreateWorkflow(); + foreach (var rc in runningContainers) + { + foreach (var c in rc.Containers) + { + DownloadContainerLog(workflow, c); + } + } + } + + private void DownloadContainerLog(IStartupWorkflow workflow, RunningContainer c) + { + var file = Log.CreateSubfile(); + Log.Log($"Downloading container log for '{c.Name}' to file '{file.FullFilename}'..."); + var handler = new LogDownloadHandler(c.Name, file); + workflow.DownloadContainerLog(c, handler); + } + //public ApplicationIds GetApplicationIds() //{ // //return new ApplicationIds( diff --git a/KubernetesWorkflow/Configuration.cs b/KubernetesWorkflow/Configuration.cs index 2c7f440..5cb7da2 100644 --- a/KubernetesWorkflow/Configuration.cs +++ b/KubernetesWorkflow/Configuration.cs @@ -15,5 +15,6 @@ public TimeSpan RetryDelay { get; } public string KubernetesNamespace { get; } public bool AllowNamespaceOverride { get; set; } = true; + public IK8sHooks Hooks { get; set; } = new DoNothingK8sHooks(); } } diff --git a/KubernetesWorkflow/K8sHooks.cs b/KubernetesWorkflow/K8sHooks.cs new file mode 100644 index 0000000..3bea4d9 --- /dev/null +++ b/KubernetesWorkflow/K8sHooks.cs @@ -0,0 +1,19 @@ +namespace KubernetesWorkflow +{ + public interface IK8sHooks + { + void OnContainersStarted(RunningContainers runningContainers); + void OnContainersStopped(RunningContainers runningContainers); + } + + public class DoNothingK8sHooks : IK8sHooks + { + public void OnContainersStarted(RunningContainers runningContainers) + { + } + + public void OnContainersStopped(RunningContainers runningContainers) + { + } + } +} diff --git a/KubernetesWorkflow/StartupWorkflow.cs b/KubernetesWorkflow/StartupWorkflow.cs index 5cc8f1e..3a95262 100644 --- a/KubernetesWorkflow/StartupWorkflow.cs +++ b/KubernetesWorkflow/StartupWorkflow.cs @@ -41,7 +41,9 @@ namespace KubernetesWorkflow if (startupConfig.CreateCrashWatcher) CreateCrashWatchers(controller, containers); - return new RunningContainers(startupConfig, runningPod, containers); + var rc = new RunningContainers(startupConfig, runningPod, containers); + cluster.Configuration.Hooks.OnContainersStarted(rc); + return rc; }); } @@ -50,6 +52,7 @@ namespace KubernetesWorkflow K8s(controller => { controller.Stop(runningContainers.RunningPod); + cluster.Configuration.Hooks.OnContainersStopped(runningContainers); }); }