cs-codex-dist-tests/DistTestCore/DistTest.cs

362 lines
12 KiB
C#
Raw Normal View History

using DistTestCore.Logs;
using FileUtils;
2023-04-14 12:53:39 +00:00
using KubernetesWorkflow;
2023-04-14 10:37:05 +00:00
using Logging;
using NUnit.Framework;
using System.Reflection;
2023-09-08 08:21:40 +00:00
using Utils;
2023-04-12 14:06:04 +00:00
namespace DistTestCore
{
2023-05-04 06:25:48 +00:00
[Parallelizable(ParallelScope.All)]
2023-09-12 09:25:04 +00:00
public abstract class DistTest : PluginInterface
2023-04-12 14:06:04 +00:00
{
2023-08-10 09:25:22 +00:00
private const string TestsType = "dist-tests";
2023-09-12 09:37:20 +00:00
private const string TestNamespacePrefix = "ct-";
2023-04-14 12:53:39 +00:00
private readonly Configuration configuration = new Configuration();
private readonly Assembly[] testAssemblies;
private readonly FixtureLog fixtureLog;
2023-07-18 07:47:44 +00:00
private readonly StatusLog statusLog;
private readonly object lifecycleLock = new object();
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
2023-09-12 08:31:55 +00:00
private readonly PluginManager PluginManager = new PluginManager();
public DistTest()
{
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
2023-07-18 07:47:44 +00:00
var logConfig = configuration.GetLogConfig();
var startTime = DateTime.UtcNow;
fixtureLog = new FixtureLog(logConfig, startTime);
2023-07-31 09:51:29 +00:00
statusLog = new StatusLog(logConfig, startTime);
}
2023-04-12 14:06:04 +00:00
[OneTimeSetUp]
public void GlobalSetup()
{
2023-09-12 08:31:55 +00:00
fixtureLog.Log($"Distributed Tests are starting...");
PluginManager.DiscoverPlugins();
AnnouncePlugins(fixtureLog);
2023-07-17 07:26:54 +00:00
2023-04-12 14:06:04 +00:00
// Previous test run may have been interrupted.
// Begin by cleaning everything up.
try
{
2023-04-14 12:53:39 +00:00
Stopwatch.Measure(fixtureLog, "Global setup", () =>
{
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), string.Empty);
2023-09-12 09:37:20 +00:00
wc.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix);
2023-04-14 12:53:39 +00:00
});
2023-04-12 14:06:04 +00:00
}
catch (Exception ex)
{
GlobalTestFailure.HasFailed = true;
fixtureLog.Error($"Global setup cleanup failed with: {ex}");
2023-04-12 14:06:04 +00:00
throw;
}
2023-04-14 12:53:39 +00:00
fixtureLog.Log("Global setup cleanup successful");
2023-04-12 14:06:04 +00:00
}
2023-09-12 08:31:55 +00:00
[OneTimeTearDown]
public void GlobalTearDown()
{
FinalizePlugins(fixtureLog);
}
2023-04-12 14:06:04 +00:00
[SetUp]
public void SetUpDistTest()
{
if (GlobalTestFailure.HasFailed)
{
Assert.Inconclusive("Skip test: Previous test failed during clean up.");
}
else
{
CreateNewTestLifecycle();
}
}
[TearDown]
public void TearDownDistTest()
{
try
{
2023-04-14 12:53:39 +00:00
DisposeTestLifecycle();
2023-04-12 14:06:04 +00:00
}
catch (Exception ex)
{
fixtureLog.Error("Cleanup failed: " + ex.Message);
2023-04-12 14:06:04 +00:00
GlobalTestFailure.HasFailed = true;
}
}
public TestFile GenerateTestFile(ByteSize size, string label = "")
2023-04-12 14:06:04 +00:00
{
return Get().FileManager.GenerateTestFile(size, label);
2023-04-12 14:06:04 +00:00
}
/// <summary>
/// Any test files generated in 'action' will be deleted after it returns.
/// This helps prevent large tests from filling up discs.
/// </summary>
public void ScopedTestFiles(Action action)
{
Get().FileManager.ScopedFiles(action);
}
//public IOnlineCodexNode SetupCodexBootstrapNode()
//{
// return SetupCodexBootstrapNode(s => { });
//}
//public virtual IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
//{
// return SetupCodexNode(s =>
// {
// setup(s);
// s.WithName("Bootstrap");
// });
//}
//public IOnlineCodexNode SetupCodexNode()
//{
// return SetupCodexNode(s => { });
//}
2023-04-25 10:52:11 +00:00
//public IOnlineCodexNode SetupCodexNode(Action<ICodexSetup> setup)
//{
// return SetupCodexNodes(1, setup)[0];
//}
2023-04-25 10:52:11 +00:00
//public ICodexNodeGroup SetupCodexNodes(int numberOfNodes)
//{
// return SetupCodexNodes(numberOfNodes, s => { });
//}
2023-04-25 10:52:11 +00:00
//public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
//{
// var codexSetup = CreateCodexSetup(numberOfNodes);
2023-04-25 10:52:11 +00:00
// setup(codexSetup);
2023-04-25 10:52:11 +00:00
// return BringOnline(codexSetup);
//}
2023-04-25 10:52:11 +00:00
//public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
//{
// return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
//}
2023-04-12 14:06:04 +00:00
//public IEnumerable<IOnlineCodexNode> GetAllOnlineCodexNodes()
//{
// return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
//}
2023-09-12 09:25:04 +00:00
public override T GetPlugin<T>()
{
return Get().GetPlugin<T>();
}
2023-09-12 08:31:55 +00:00
private void AnnouncePlugins(FixtureLog fixtureLog)
{
PluginManager.AnnouncePlugins(fixtureLog);
}
private void FinalizePlugins(FixtureLog fixtureLog)
{
PluginManager.FinalizePlugins(fixtureLog);
}
public ILog GetTestLog()
{
return Get().Log;
}
public void Log(string msg)
2023-04-26 12:40:54 +00:00
{
TestContext.Progress.WriteLine(msg);
GetTestLog().Log(msg);
}
public void Debug(string msg)
{
TestContext.Progress.WriteLine(msg);
GetTestLog().Debug(msg);
}
//public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers()
//{
// return new PeerConnectionTestHelpers(GetTestLog());
//}
2023-08-24 08:59:11 +00:00
//public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers()
//{
// return new PeerDownloadTestHelpers(GetTestLog(), Get().FileManager);
//}
2023-08-24 08:59:11 +00:00
2023-08-22 13:51:39 +00:00
public void Measure(string name, Action action)
{
Stopwatch.Measure(Get().Log, name, action);
}
//protected CodexSetup CreateCodexSetup(int numberOfNodes)
//{
// return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
//}
2023-05-31 11:34:12 +00:00
2023-08-22 13:51:39 +00:00
private TestLifecycle Get()
{
lock (lifecycleLock)
{
return lifecycles[GetCurrentTestName()];
}
2023-04-26 12:40:54 +00:00
}
2023-04-12 14:06:04 +00:00
private void CreateNewTestLifecycle()
{
2023-05-03 12:18:37 +00:00
var testName = GetCurrentTestName();
2023-07-18 12:26:21 +00:00
fixtureLog.WriteLogTag();
2023-05-03 12:18:37 +00:00
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
2023-04-14 12:53:39 +00:00
{
lock (lifecycleLock)
{
2023-09-12 09:37:20 +00:00
var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString();
var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace);
lifecycles.Add(testName, lifecycle);
DefaultContainerRecipe.TestsType = TestsType;
2023-09-12 08:31:55 +00:00
//DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
}
2023-04-14 12:53:39 +00:00
});
}
private void DisposeTestLifecycle()
{
var lifecycle = Get();
2023-07-21 07:20:28 +00:00
var testResult = GetTestResult();
var testDuration = lifecycle.GetTestDuration();
fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
2023-09-12 08:31:55 +00:00
statusLog.ConcludeTest(testResult, testDuration);//, lifecycle.GetApplicationIds());
2023-04-14 12:53:39 +00:00
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
2023-09-12 08:31:55 +00:00
WriteEndTestLog(lifecycle.Log);
IncludeLogsAndMetricsOnTestFailure(lifecycle);
2023-04-14 12:53:39 +00:00
lifecycle.DeleteAllResources();
lifecycle = null!;
});
}
2023-09-12 08:31:55 +00:00
private void WriteEndTestLog(TestLog log)
{
var result = TestContext.CurrentContext.Result;
Log($"*** Finished: {GetCurrentTestName()} = {result.Outcome.Status}");
if (!string.IsNullOrEmpty(result.Message))
{
Log(result.Message);
Log($"{result.StackTrace}");
}
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
log.MarkAsFailed();
}
}
private ITimeSet GetTimeSet()
{
if (ShouldUseLongTimeouts()) return new LongTimeSet();
return new DefaultTimeSet();
}
private bool ShouldUseLongTimeouts()
{
// Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
// But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
// if the attribute is present.
var currentTest = TestContext.CurrentContext.Test;
var className = currentTest.ClassName;
var methodName = currentTest.MethodName;
var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
}
private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
{
var result = TestContext.CurrentContext.Result;
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
{
fixtureLog.MarkAsFailed();
if (IsDownloadingLogsAndMetricsEnabled())
{
lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
//DownloadAllLogs(lifecycle);
//DownloadAllMetrics(lifecycle);
}
else
{
lifecycle.Log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
}
}
}
//private void DownloadAllLogs(TestLifecycle lifecycle)
//{
// OnEachCodexNode(lifecycle, node =>
// {
// lifecycle.DownloadLog(node.CodexAccess.Container);
// });
//}
//private void DownloadAllMetrics(TestLifecycle lifecycle)
//{
// var metricsDownloader = new MetricsDownloader(lifecycle.Log);
// OnEachCodexNode(lifecycle, node =>
// {
// var m = node.Metrics as MetricsAccess;
// if (m != null)
// {
// metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m);
// }
// });
//}
//private void OnEachCodexNode(TestLifecycle lifecycle, Action<OnlineCodexNode> action)
//{
// var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
// foreach (var node in allNodes)
// {
// action(node);
// }
//}
2023-04-12 14:06:04 +00:00
2023-04-14 12:53:39 +00:00
private string GetCurrentTestName()
{
return $"[{TestContext.CurrentContext.Test.Name}]";
}
private string GetTestResult()
{
return TestContext.CurrentContext.Result.Outcome.Status.ToString();
}
private bool IsDownloadingLogsAndMetricsEnabled()
{
var testProperties = TestContext.CurrentContext.Test.Properties;
return !testProperties.ContainsKey(DontDownloadLogsAndMetricsOnFailureAttribute.DontDownloadKey);
}
2023-04-12 14:06:04 +00:00
}
public static class GlobalTestFailure
{
public static bool HasFailed { get; set; } = false;
}
}