status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Automatic downloading of container logs on test failure

This commit is contained in:
benbierens 2023-09-13 15:10:19 +02:00
parent ca5981e852
commit a2e07fbd2e
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
8 changed files with 88 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
DistTestCore/Configuration.cs
View File

@ -1,4 +1,5 @@
using Core;
using KubernetesWorkflow;
namespace DistTestCore
{
@ -26,6 +27,11 @@ namespace DistTestCore
}
public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace)
{
return GetK8sConfiguration(timeSet, new DoNothingK8sHooks(), k8sNamespace);
}
public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, IK8sHooks hooks, string k8sNamespace)
{
var config = new KubernetesWorkflow.Configuration(
kubeConfigFile: kubeConfigFile,
@ -35,6 +41,7 @@ namespace DistTestCore
);
config.AllowNamespaceOverride = false;
config.Hooks = hooks;
return config;
}

17
DistTestCore/DistTest.cs
View File

@ -171,7 +171,7 @@ namespace DistTestCore
{
WriteEndTestLog(lifecycle.Log);
IncludeLogsAndMetricsOnTestFailure(lifecycle);
IncludeLogsOnTestFailure(lifecycle);
lifecycle.DeleteAllResources();
lifecycle = null!;
});
@ -215,22 +215,21 @@ namespace DistTestCore
return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
}
private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
private void IncludeLogsOnTestFailure(TestLifecycle lifecycle)
{
var result = TestContext.CurrentContext.Result;
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
fixtureLog.MarkAsFailed();
if (IsDownloadingLogsAndMetricsEnabled())
if (IsDownloadingLogsEnabled())
{
lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
//DownloadAllLogs(lifecycle);
//DownloadAllMetrics(lifecycle);
lifecycle.Log.Log("Downloading all container logs because of test failure...");
lifecycle.DownloadAllLogs();
}
else
{
lifecycle.Log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
lifecycle.Log.Log("Skipping download of all container logs due to [DontDownloadLogsOnFailure] attribute.");
}
}
}
@ -245,10 +244,10 @@ namespace DistTestCore
return TestContext.CurrentContext.Result.Outcome.Status.ToString();
}
private bool IsDownloadingLogsAndMetricsEnabled()
private bool IsDownloadingLogsEnabled()
{
var testProperties = TestContext.CurrentContext.Test.Properties;
return !testProperties.ContainsKey(DontDownloadLogsAndMetricsOnFailureAttribute.DontDownloadKey);
return !testProperties.ContainsKey(DontDownloadLogsOnFailureAttribute.DontDownloadKey);
}
}

15
DistTestCore/DontDownloadLogsAndMetricsOnFailureAttribute.cs
View File

@ -1,15 +0,0 @@
using NUnit.Framework;
namespace DistTestCore
{
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public class DontDownloadLogsAndMetricsOnFailureAttribute : PropertyAttribute
{
public const string DontDownloadKey = "DontDownloadLogsAndMetrics";
public DontDownloadLogsAndMetricsOnFailureAttribute()
: base(DontDownloadKey)
{
}
}
}

15
DistTestCore/DontDownloadLogsOnFailureAttribute.cs Normal file
View File

@ -0,0 +1,15 @@
using NUnit.Framework;
namespace DistTestCore
{
[AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
public class DontDownloadLogsOnFailureAttribute : PropertyAttribute
{
public const string DontDownloadKey = "DontDownloadLogs";
public DontDownloadLogsOnFailureAttribute()
: base(DontDownloadKey)
{
}
}
}

36
DistTestCore/TestLifecycle.cs
View File

@ -1,14 +1,16 @@
using Core;
using FileUtils;
using KubernetesWorkflow;
using Logging;
using Utils;
namespace DistTestCore
{
public class TestLifecycle
public class TestLifecycle : IK8sHooks
{
private readonly DateTime testStart;
private readonly EntryPoint entryPoint;
private readonly List<RunningContainers> runningContainers = new List<RunningContainers>();
public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, string testNamespace)
{
@ -17,7 +19,7 @@ namespace DistTestCore
TimeSet = timeSet;
testStart = DateTime.UtcNow;
entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, testNamespace), configuration.GetFileManagerFolder(), timeSet);
entryPoint = new EntryPoint(log, configuration.GetK8sConfiguration(timeSet, this, testNamespace), configuration.GetFileManagerFolder(), timeSet);
CoreInterface = entryPoint.CreateInterface();
log.WriteLogTag();
@ -51,6 +53,36 @@ namespace DistTestCore
return Time.FormatDuration(testDuration);
}
public void OnContainersStarted(RunningContainers rc)
{
runningContainers.Add(rc);
}
public void OnContainersStopped(RunningContainers rc)
{
runningContainers.Remove(rc);
}
public void DownloadAllLogs()
{
var workflow = entryPoint.Tools.CreateWorkflow();
foreach (var rc in runningContainers)
{
foreach (var c in rc.Containers)
{
DownloadContainerLog(workflow, c);
}
}
}
private void DownloadContainerLog(IStartupWorkflow workflow, RunningContainer c)
{
var file = Log.CreateSubfile();
Log.Log($"Downloading container log for '{c.Name}' to file '{file.FullFilename}'...");
var handler = new LogDownloadHandler(c.Name, file);
workflow.DownloadContainerLog(c, handler);
}
//public ApplicationIds GetApplicationIds()
//{
// //return new ApplicationIds(

1
KubernetesWorkflow/Configuration.cs
View File

@ -15,5 +15,6 @@
public TimeSpan RetryDelay { get; }
public string KubernetesNamespace { get; }
public bool AllowNamespaceOverride { get; set; } = true;
public IK8sHooks Hooks { get; set; } = new DoNothingK8sHooks();
}
}

19
KubernetesWorkflow/K8sHooks.cs Normal file
View File

@ -0,0 +1,19 @@
namespace KubernetesWorkflow
{
public interface IK8sHooks
{
void OnContainersStarted(RunningContainers runningContainers);
void OnContainersStopped(RunningContainers runningContainers);
}
public class DoNothingK8sHooks : IK8sHooks
{
public void OnContainersStarted(RunningContainers runningContainers)
{
}
public void OnContainersStopped(RunningContainers runningContainers)
{
}
}
}

5
KubernetesWorkflow/StartupWorkflow.cs
View File

@ -41,7 +41,9 @@ namespace KubernetesWorkflow
if (startupConfig.CreateCrashWatcher) CreateCrashWatchers(controller, containers);
return new RunningContainers(startupConfig, runningPod, containers);
var rc = new RunningContainers(startupConfig, runningPod, containers);
cluster.Configuration.Hooks.OnContainersStarted(rc);
return rc;
});
}
@ -50,6 +52,7 @@ namespace KubernetesWorkflow
K8s(controller =>
{
controller.Stop(runningContainers.RunningPod);
cluster.Configuration.Hooks.OnContainersStopped(runningContainers);
});
}