cs-codex-dist-tests/DistTestCore/DistTest.cs

307 lines
10 KiB
C#
Raw Normal View History

2023-04-14 10:37:05 +00:00
using DistTestCore.Codex;
using DistTestCore.Helpers;
2023-04-14 10:37:05 +00:00
using DistTestCore.Logs;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
2023-04-14 12:53:39 +00:00
using KubernetesWorkflow;
2023-04-14 10:37:05 +00:00
using Logging;
using NUnit.Framework;
using System.Reflection;
2023-04-12 14:06:04 +00:00
namespace DistTestCore
{
2023-05-04 06:25:48 +00:00
[Parallelizable(ParallelScope.All)]
2023-04-12 14:06:04 +00:00
public abstract class DistTest
{
2023-04-14 12:53:39 +00:00
private readonly Configuration configuration = new Configuration();
private readonly Assembly[] testAssemblies;
private readonly FixtureLog fixtureLog;
2023-07-18 07:47:44 +00:00
private readonly StatusLog statusLog;
private readonly object lifecycleLock = new object();
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
2023-04-12 14:06:04 +00:00
public DistTest()
{
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
2023-07-18 07:47:44 +00:00
var logConfig = configuration.GetLogConfig();
var startTime = DateTime.UtcNow;
fixtureLog = new FixtureLog(logConfig, startTime);
statusLog = new StatusLog(logConfig, startTime, CodexContainerRecipe.DockerImage);
PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
}
public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; }
public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; }
2023-04-12 14:06:04 +00:00
[OneTimeSetUp]
public void GlobalSetup()
{
// Previous test run may have been interrupted.
// Begin by cleaning everything up.
try
{
2023-04-14 12:53:39 +00:00
Stopwatch.Measure(fixtureLog, "Global setup", () =>
{
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()));
2023-04-14 12:53:39 +00:00
wc.CreateWorkflow().DeleteAllResources();
});
2023-04-12 14:06:04 +00:00
}
catch (Exception ex)
{
GlobalTestFailure.HasFailed = true;
fixtureLog.Error($"Global setup cleanup failed with: {ex}");
2023-04-12 14:06:04 +00:00
throw;
}
2023-04-14 12:53:39 +00:00
fixtureLog.Log("Global setup cleanup successful");
fixtureLog.Log($"Codex image: '{CodexContainerRecipe.DockerImage}'");
fixtureLog.Log($"Prometheus image: '{PrometheusContainerRecipe.DockerImage}'");
fixtureLog.Log($"Geth image: '{GethContainerRecipe.DockerImage}'");
2023-04-12 14:06:04 +00:00
}
[SetUp]
public void SetUpDistTest()
{
if (GlobalTestFailure.HasFailed)
{
Assert.Inconclusive("Skip test: Previous test failed during clean up.");
}
else
{
CreateNewTestLifecycle();
}
}
[TearDown]
public void TearDownDistTest()
{
try
{
2023-04-14 12:53:39 +00:00
DisposeTestLifecycle();
2023-04-12 14:06:04 +00:00
}
catch (Exception ex)
{
fixtureLog.Error("Cleanup failed: " + ex.Message);
2023-04-12 14:06:04 +00:00
GlobalTestFailure.HasFailed = true;
}
}
public TestFile GenerateTestFile(ByteSize size, string label = "")
2023-04-12 14:06:04 +00:00
{
return Get().FileManager.GenerateTestFile(size, label);
2023-04-12 14:06:04 +00:00
}
/// <summary>
/// Any test files generated in 'action' will be deleted after it returns.
/// This helps prevent large tests from filling up discs.
/// </summary>
public void ScopedTestFiles(Action action)
{
Get().FileManager.PushFileSet();
action();
Get().FileManager.PopFileSet();
}
public IOnlineCodexNode SetupCodexBootstrapNode()
{
return SetupCodexBootstrapNode(s => { });
}
public virtual IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
{
return SetupCodexNode(s =>
{
setup(s);
s.WithName("Bootstrap");
});
}
2023-04-25 10:52:11 +00:00
public IOnlineCodexNode SetupCodexNode()
2023-04-12 14:06:04 +00:00
{
2023-04-25 10:52:11 +00:00
return SetupCodexNode(s => { });
}
public IOnlineCodexNode SetupCodexNode(Action<ICodexSetup> setup)
{
return SetupCodexNodes(1, setup)[0];
}
public ICodexNodeGroup SetupCodexNodes(int numberOfNodes)
{
return SetupCodexNodes(numberOfNodes, s => { });
}
public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
2023-04-25 10:52:11 +00:00
{
2023-05-31 11:34:12 +00:00
var codexSetup = CreateCodexSetup(numberOfNodes);
2023-04-25 10:52:11 +00:00
setup(codexSetup);
return BringOnline(codexSetup);
}
public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
{
return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
2023-04-12 14:06:04 +00:00
}
public IEnumerable<IOnlineCodexNode> GetAllOnlineCodexNodes()
{
return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
}
public BaseLog GetTestLog()
{
return Get().Log;
}
public void Log(string msg)
2023-04-26 12:40:54 +00:00
{
TestContext.Progress.WriteLine(msg);
GetTestLog().Log(msg);
}
public void Debug(string msg)
{
TestContext.Progress.WriteLine(msg);
GetTestLog().Debug(msg);
}
2023-05-31 11:34:12 +00:00
protected CodexSetup CreateCodexSetup(int numberOfNodes)
{
return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
}
private TestLifecycle Get()
{
lock (lifecycleLock)
{
return lifecycles[GetCurrentTestName()];
}
2023-04-26 12:40:54 +00:00
}
2023-04-12 14:06:04 +00:00
private void CreateNewTestLifecycle()
{
2023-05-03 12:18:37 +00:00
var testName = GetCurrentTestName();
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
2023-04-14 12:53:39 +00:00
{
lock (lifecycleLock)
{
lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet()));
}
2023-04-14 12:53:39 +00:00
});
}
private void DisposeTestLifecycle()
{
var lifecycle = Get();
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
2023-07-18 07:47:44 +00:00
statusLog.ConcludeTest(GetTestResult());
2023-04-14 12:53:39 +00:00
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
lifecycle.Log.EndTest();
IncludeLogsAndMetricsOnTestFailure(lifecycle);
2023-04-14 12:53:39 +00:00
lifecycle.DeleteAllResources();
lifecycle = null!;
});
}
private ITimeSet GetTimeSet()
{
if (ShouldUseLongTimeouts()) return new LongTimeSet();
return new DefaultTimeSet();
}
private bool ShouldUseLongTimeouts()
{
// Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
// But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
// if the attribute is present.
var currentTest = TestContext.CurrentContext.Test;
var className = currentTest.ClassName;
var methodName = currentTest.MethodName;
var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
}
private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
{
var result = TestContext.CurrentContext.Result;
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
fixtureLog.MarkAsFailed();
if (IsDownloadingLogsAndMetricsEnabled())
{
lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
DownloadAllLogs(lifecycle);
DownloadAllMetrics(lifecycle);
}
else
{
lifecycle.Log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
}
}
}
private void DownloadAllLogs(TestLifecycle lifecycle)
2023-04-12 14:06:04 +00:00
{
OnEachCodexNode(lifecycle, node =>
{
2023-06-27 13:28:00 +00:00
lifecycle.DownloadLog(node.CodexAccess.Container);
});
}
private void DownloadAllMetrics(TestLifecycle lifecycle)
{
var metricsDownloader = new MetricsDownloader(lifecycle.Log);
OnEachCodexNode(lifecycle, node =>
{
var m = node.Metrics as MetricsAccess;
if (m != null)
{
metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m);
}
});
}
private void OnEachCodexNode(TestLifecycle lifecycle, Action<OnlineCodexNode> action)
{
var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
foreach (var node in allNodes)
{
action(node);
}
2023-04-12 14:06:04 +00:00
}
2023-04-14 12:53:39 +00:00
private string GetCurrentTestName()
{
return $"[{TestContext.CurrentContext.Test.Name}]";
}
private string GetTestResult()
{
return TestContext.CurrentContext.Result.Outcome.Status.ToString();
}
private bool IsDownloadingLogsAndMetricsEnabled()
{
var testProperties = TestContext.CurrentContext.Test.Properties;
return !testProperties.ContainsKey(DontDownloadLogsAndMetricsOnFailureAttribute.DontDownloadKey);
}
2023-04-12 14:06:04 +00:00
}
public static class GlobalTestFailure
{
public static bool HasFailed { get; set; } = false;
}
}