Automatic downloading of container logs on test failure

DistTestCore/Configuration.cs

@@ -1,4 +1,5 @@
 using Core;
+using KubernetesWorkflow;
 
 namespace DistTestCore
 {
@@ -26,6 +27,11 @@ namespace DistTestCore
         }
 
         public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace)
+        {
+            return GetK8sConfiguration(timeSet, new DoNothingK8sHooks(), k8sNamespace);
+        }
+
+        public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, IK8sHooks hooks, string k8sNamespace)
         {
             var config = new KubernetesWorkflow.Configuration(
                 kubeConfigFile: kubeConfigFile,
@@ -35,6 +41,7 @@ namespace DistTestCore
             );
 
             config.AllowNamespaceOverride = false;
+            config.Hooks = hooks;
 
             return config;
         }

DistTestCore/DistTest.cs

@@ -171,7 +171,7 @@ namespace DistTestCore
             {
                 WriteEndTestLog(lifecycle.Log);
 
-                IncludeLogsAndMetricsOnTestFailure(lifecycle);
+                IncludeLogsOnTestFailure(lifecycle);
                 lifecycle.DeleteAllResources();
                 lifecycle = null!;
             });
@@ -215,22 +215,21 @@ namespace DistTestCore
             return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
         }
 
-        private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
+        private void IncludeLogsOnTestFailure(TestLifecycle lifecycle)
         {
             var result = TestContext.CurrentContext.Result;
             if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
             {
                 fixtureLog.MarkAsFailed();
 
-                if (IsDownloadingLogsAndMetricsEnabled())
+                if (IsDownloadingLogsEnabled())
                 {
-                    lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
+                    lifecycle.Log.Log("Downloading all container logs because of test failure...");
-                    //DownloadAllLogs(lifecycle);
+                    lifecycle.DownloadAllLogs();
-                    //DownloadAllMetrics(lifecycle);
                 }
                 else
                 {
-                    lifecycle.Log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
+                    lifecycle.Log.Log("Skipping download of all container logs due to [DontDownloadLogsOnFailure] attribute.");
                 }
             }
         }
@@ -245,10 +244,10 @@ namespace DistTestCore
             return TestContext.CurrentContext.Result.Outcome.Status.ToString();
         }
 
-        private bool IsDownloadingLogsAndMetricsEnabled()
+        private bool IsDownloadingLogsEnabled()
         {
             var testProperties = TestContext.CurrentContext.Test.Properties;
-            return !testProperties.ContainsKey(DontDownloadLogsAndMetricsOnFailureAttribute.DontDownloadKey);
+            return !testProperties.ContainsKey(DontDownloadLogsOnFailureAttribute.DontDownloadKey);
         }
     }
 

DistTestCore/DontDownloadLogsAndMetricsOnFailureAttribute.cs

@@ -1,15 +0,0 @@
-using NUnit.Framework;
-
-namespace DistTestCore
-{
-    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
-    public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute
-    {
-        public const string DontDownloadKey = "DontDownloadLogsAndMetrics";
-
-        public DontDownloadLogsAndMetricsOnFailureAttribute()
-            : base(DontDownloadKey)
-        {
-        }
-    }
-}

DistTestCore/DontDownloadLogsOnFailureAttribute.cs

@@ -0,0 +1,15 @@
+using NUnit.Framework;
+
+namespace DistTestCore
+{
+    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+    public class DontDownloadLogsOnFailureAttribute : PropertyAttribute
+    {
+        public const string DontDownloadKey = "DontDownloadLogs";
+
+        public DontDownloadLogsOnFailureAttribute()
+            : base(DontDownloadKey)
+        {
+        }
+    }
+}

DistTestCore/TestLifecycle.cs

@@ -1,14 +1,16 @@
 using Core;
 using FileUtils;
+using KubernetesWorkflow;
 using Logging;
 using Utils;
 
 namespace DistTestCore
 {
-    public class TestLifecycle
+    public class TestLifecycle : IK8sHooks
     {
         private readonly DateTime testStart;
         private readonly EntryPoint entryPoint;
+        private readonly List<RunningContainers> runningContainers = new List<RunningContainers>();
 
         public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
         {
@@ -17,7 +19,7 @@ namespace DistTestCore
             TimeSet = timeSet;
             testStart = DateTime.UtcNow;
 
-            entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, testNamespace), configuration.GetFileManagerFolder(), timeSet);
+            entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet);
             CoreInterface = entryPoint.CreateInterface();
 
             log.WriteLogTag();
@@ -51,6 +53,36 @@ namespace DistTestCore
             return Time.FormatDuration(testDuration);
         }
 
+        public void OnContainersStarted(RunningContainers rc)
+        {
+            runningContainers.Add(rc);
+        }
+
+        public void OnContainersStopped(RunningContainers rc)
+        {
+            runningContainers.Remove(rc);
+        }
+
+        public void DownloadAllLogs()
+        {
+            var workflow = entryPoint.Tools.CreateWorkflow();
+            foreach (var rc in runningContainers)
+            {
+                foreach (var c in rc.Containers)
+                {
+                    DownloadContainerLog(workflow, c);
+                }
+            }
+        }
+
+        private void DownloadContainerLog(IStartupWorkflow workflow, RunningContainer c)
+        {
+            var file = Log.CreateSubfile();
+            Log.Log($"Downloading container log for '{c.Name}' to file '{file.FullFilename}'...");
+            var handler = new LogDownloadHandler(c.Name, file);
+            workflow.DownloadContainerLog(c, handler);
+        }
+
         //public ApplicationIds GetApplicationIds()
         //{
         //    //return new ApplicationIds(

KubernetesWorkflow/Configuration.cs

@@ -15,5 +15,6 @@
         public TimeSpan RetryDelay { get; }
         public string KubernetesNamespace { get; }
         public bool AllowNamespaceOverride { get; set; } = true;
+        public IK8sHooks Hooks { get; set; } = new DoNothingK8sHooks();
     }
 }

KubernetesWorkflow/K8sHooks.cs

@@ -0,0 +1,19 @@
+namespace KubernetesWorkflow
+{
+    public interface IK8sHooks
+    {
+        void OnContainersStarted(RunningContainers runningContainers);
+        void OnContainersStopped(RunningContainers runningContainers);
+    }
+
+    public class DoNothingK8sHooks : IK8sHooks
+    {
+        public void OnContainersStarted(RunningContainers runningContainers)
+        {
+        }
+
+        public void OnContainersStopped(RunningContainers runningContainers)
+        {
+        }
+    }
+}

KubernetesWorkflow/StartupWorkflow.cs

@@ -41,7 +41,9 @@ namespace KubernetesWorkflow
 
                 if (startupConfig.CreateCrashWatcher) CreateCrashWatchers(controller, containers);
 
-                return new RunningContainers(startupConfig, runningPod, containers);
+                var rc = new RunningContainers(startupConfig, runningPod, containers);
+                cluster.Configuration.Hooks.OnContainersStarted(rc);
+                return rc;
             });
         }
 
@@ -50,6 +52,7 @@ namespace KubernetesWorkflow
             K8s(controller =>
             {
                 controller.Stop(runningContainers.RunningPod);
+                cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
             });
         }