2023-09-11 09:59:33 +00:00
|
|
|
|
using DistTestCore.Logs;
|
2023-09-11 08:43:27 +00:00
|
|
|
|
using FileUtils;
|
2023-04-14 12:53:39 +00:00
|
|
|
|
using KubernetesWorkflow;
|
2023-04-14 10:37:05 +00:00
|
|
|
|
using Logging;
|
2023-04-13 09:53:54 +00:00
|
|
|
|
using NUnit.Framework;
|
2023-04-27 13:55:33 +00:00
|
|
|
|
using System.Reflection;
|
2023-09-08 08:21:40 +00:00
|
|
|
|
using Utils;
|
2023-04-12 14:06:04 +00:00
|
|
|
|
|
|
|
|
|
namespace DistTestCore
|
|
|
|
|
{
|
2023-05-04 06:25:48 +00:00
|
|
|
|
[Parallelizable(ParallelScope.All)]
|
2023-09-12 09:25:04 +00:00
|
|
|
|
public abstract class DistTest : PluginInterface
|
2023-04-12 14:06:04 +00:00
|
|
|
|
{
|
2023-08-10 09:25:22 +00:00
|
|
|
|
private const string TestsType = "dist-tests";
|
2023-09-12 09:37:20 +00:00
|
|
|
|
private const string TestNamespacePrefix = "ct-";
|
2023-04-14 12:53:39 +00:00
|
|
|
|
private readonly Configuration configuration = new Configuration();
|
2023-04-27 13:55:33 +00:00
|
|
|
|
private readonly Assembly[] testAssemblies;
|
2023-05-03 12:55:26 +00:00
|
|
|
|
private readonly FixtureLog fixtureLog;
|
2023-07-18 07:47:44 +00:00
|
|
|
|
private readonly StatusLog statusLog;
|
2023-05-03 12:55:26 +00:00
|
|
|
|
private readonly object lifecycleLock = new object();
|
|
|
|
|
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
|
2023-09-12 08:31:55 +00:00
|
|
|
|
private readonly PluginManager PluginManager = new PluginManager();
|
|
|
|
|
|
2023-04-27 13:55:33 +00:00
|
|
|
|
public DistTest()
|
|
|
|
|
{
|
|
|
|
|
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
|
|
|
|
|
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
|
2023-05-03 12:55:26 +00:00
|
|
|
|
|
2023-07-18 07:47:44 +00:00
|
|
|
|
var logConfig = configuration.GetLogConfig();
|
|
|
|
|
var startTime = DateTime.UtcNow;
|
|
|
|
|
fixtureLog = new FixtureLog(logConfig, startTime);
|
2023-07-31 09:51:29 +00:00
|
|
|
|
statusLog = new StatusLog(logConfig, startTime);
|
2023-04-27 13:55:33 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-04-12 14:06:04 +00:00
|
|
|
|
[OneTimeSetUp]
|
|
|
|
|
public void GlobalSetup()
|
|
|
|
|
{
|
2023-09-12 08:31:55 +00:00
|
|
|
|
fixtureLog.Log($"Distributed Tests are starting...");
|
|
|
|
|
PluginManager.DiscoverPlugins();
|
|
|
|
|
AnnouncePlugins(fixtureLog);
|
2023-07-17 07:26:54 +00:00
|
|
|
|
|
2023-04-12 14:06:04 +00:00
|
|
|
|
// Previous test run may have been interrupted.
|
|
|
|
|
// Begin by cleaning everything up.
|
|
|
|
|
try
|
|
|
|
|
{
|
2023-04-14 12:53:39 +00:00
|
|
|
|
Stopwatch.Measure(fixtureLog, "Global setup", () =>
|
|
|
|
|
{
|
2023-09-04 07:08:34 +00:00
|
|
|
|
var wc = new WorkflowCreator(fixtureLog, configuration.GetK8sConfiguration(GetTimeSet()), string.Empty);
|
2023-09-12 09:37:20 +00:00
|
|
|
|
wc.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix);
|
2023-04-14 12:53:39 +00:00
|
|
|
|
});
|
2023-04-12 14:06:04 +00:00
|
|
|
|
}
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
{
|
|
|
|
|
GlobalTestFailure.HasFailed = true;
|
2023-04-17 14:28:07 +00:00
|
|
|
|
fixtureLog.Error($"Global setup cleanup failed with: {ex}");
|
2023-04-12 14:06:04 +00:00
|
|
|
|
throw;
|
|
|
|
|
}
|
2023-04-14 12:53:39 +00:00
|
|
|
|
|
|
|
|
|
fixtureLog.Log("Global setup cleanup successful");
|
2023-04-12 14:06:04 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-09-12 08:31:55 +00:00
|
|
|
|
[OneTimeTearDown]
|
|
|
|
|
public void GlobalTearDown()
|
|
|
|
|
{
|
|
|
|
|
FinalizePlugins(fixtureLog);
|
|
|
|
|
}
|
|
|
|
|
|
2023-04-12 14:06:04 +00:00
|
|
|
|
[SetUp]
|
|
|
|
|
public void SetUpDistTest()
|
|
|
|
|
{
|
|
|
|
|
if (GlobalTestFailure.HasFailed)
|
|
|
|
|
{
|
|
|
|
|
Assert.Inconclusive("Skip test: Previous test failed during clean up.");
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
CreateNewTestLifecycle();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
[TearDown]
|
|
|
|
|
public void TearDownDistTest()
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
2023-04-14 12:53:39 +00:00
|
|
|
|
DisposeTestLifecycle();
|
2023-04-12 14:06:04 +00:00
|
|
|
|
}
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
{
|
2023-04-17 14:28:07 +00:00
|
|
|
|
fixtureLog.Error("Cleanup failed: " + ex.Message);
|
2023-04-12 14:06:04 +00:00
|
|
|
|
GlobalTestFailure.HasFailed = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-06-04 06:59:51 +00:00
|
|
|
|
public TestFile GenerateTestFile(ByteSize size, string label = "")
|
2023-04-12 14:06:04 +00:00
|
|
|
|
{
|
2023-06-04 06:59:51 +00:00
|
|
|
|
return Get().FileManager.GenerateTestFile(size, label);
|
2023-04-12 14:06:04 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-05-29 07:13:38 +00:00
|
|
|
|
/// <summary>
|
|
|
|
|
/// Any test files generated in 'action' will be deleted after it returns.
|
|
|
|
|
/// This helps prevent large tests from filling up discs.
|
|
|
|
|
/// </summary>
|
|
|
|
|
public void ScopedTestFiles(Action action)
|
|
|
|
|
{
|
2023-09-11 08:43:27 +00:00
|
|
|
|
Get().FileManager.ScopedFiles(action);
|
2023-05-29 07:13:38 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
//public IOnlineCodexNode SetupCodexBootstrapNode()
|
|
|
|
|
//{
|
|
|
|
|
// return SetupCodexBootstrapNode(s => { });
|
|
|
|
|
//}
|
2023-04-30 08:56:19 +00:00
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
//public virtual IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
|
|
|
|
|
//{
|
|
|
|
|
// return SetupCodexNode(s =>
|
|
|
|
|
// {
|
|
|
|
|
// setup(s);
|
|
|
|
|
// s.WithName("Bootstrap");
|
|
|
|
|
// });
|
|
|
|
|
//}
|
2023-04-30 08:56:19 +00:00
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
//public IOnlineCodexNode SetupCodexNode()
|
|
|
|
|
//{
|
|
|
|
|
// return SetupCodexNode(s => { });
|
|
|
|
|
//}
|
2023-04-25 10:52:11 +00:00
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
//public IOnlineCodexNode SetupCodexNode(Action<ICodexSetup> setup)
|
|
|
|
|
//{
|
|
|
|
|
// return SetupCodexNodes(1, setup)[0];
|
|
|
|
|
//}
|
2023-04-25 10:52:11 +00:00
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
//public ICodexNodeGroup SetupCodexNodes(int numberOfNodes)
|
|
|
|
|
//{
|
|
|
|
|
// return SetupCodexNodes(numberOfNodes, s => { });
|
|
|
|
|
//}
|
2023-04-25 10:52:11 +00:00
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
//public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
|
|
|
|
|
//{
|
|
|
|
|
// var codexSetup = CreateCodexSetup(numberOfNodes);
|
2023-04-25 10:52:11 +00:00
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
// setup(codexSetup);
|
2023-04-25 10:52:11 +00:00
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
// return BringOnline(codexSetup);
|
|
|
|
|
//}
|
2023-04-25 10:52:11 +00:00
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
//public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
|
|
|
|
|
//{
|
|
|
|
|
// return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
|
|
|
|
|
//}
|
2023-04-12 14:06:04 +00:00
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
//public IEnumerable<IOnlineCodexNode> GetAllOnlineCodexNodes()
|
|
|
|
|
//{
|
|
|
|
|
// return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
|
|
|
|
|
//}
|
2023-05-10 07:55:36 +00:00
|
|
|
|
|
2023-09-12 09:25:04 +00:00
|
|
|
|
public override T GetPlugin<T>()
|
|
|
|
|
{
|
|
|
|
|
return Get().GetPlugin<T>();
|
|
|
|
|
}
|
|
|
|
|
|
2023-09-12 08:31:55 +00:00
|
|
|
|
private void AnnouncePlugins(FixtureLog fixtureLog)
|
|
|
|
|
{
|
|
|
|
|
PluginManager.AnnouncePlugins(fixtureLog);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void FinalizePlugins(FixtureLog fixtureLog)
|
|
|
|
|
{
|
|
|
|
|
PluginManager.FinalizePlugins(fixtureLog);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public ILog GetTestLog()
|
2023-05-10 07:55:36 +00:00
|
|
|
|
{
|
|
|
|
|
return Get().Log;
|
|
|
|
|
}
|
|
|
|
|
|
2023-05-29 07:13:38 +00:00
|
|
|
|
public void Log(string msg)
|
2023-04-26 12:40:54 +00:00
|
|
|
|
{
|
2023-05-04 12:55:39 +00:00
|
|
|
|
TestContext.Progress.WriteLine(msg);
|
2023-05-10 07:55:36 +00:00
|
|
|
|
GetTestLog().Log(msg);
|
2023-05-04 12:55:39 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-05-29 07:13:38 +00:00
|
|
|
|
public void Debug(string msg)
|
2023-05-04 12:55:39 +00:00
|
|
|
|
{
|
|
|
|
|
TestContext.Progress.WriteLine(msg);
|
2023-05-10 07:55:36 +00:00
|
|
|
|
GetTestLog().Debug(msg);
|
2023-05-03 12:55:26 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
//public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers()
|
|
|
|
|
//{
|
|
|
|
|
// return new PeerConnectionTestHelpers(GetTestLog());
|
|
|
|
|
//}
|
2023-08-24 08:59:11 +00:00
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
//public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers()
|
|
|
|
|
//{
|
|
|
|
|
// return new PeerDownloadTestHelpers(GetTestLog(), Get().FileManager);
|
|
|
|
|
//}
|
2023-08-24 08:59:11 +00:00
|
|
|
|
|
2023-08-22 13:51:39 +00:00
|
|
|
|
public void Measure(string name, Action action)
|
|
|
|
|
{
|
|
|
|
|
Stopwatch.Measure(Get().Log, name, action);
|
|
|
|
|
}
|
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
//protected CodexSetup CreateCodexSetup(int numberOfNodes)
|
|
|
|
|
//{
|
|
|
|
|
// return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
|
|
|
|
|
//}
|
2023-05-31 11:34:12 +00:00
|
|
|
|
|
2023-08-22 13:51:39 +00:00
|
|
|
|
private TestLifecycle Get()
|
2023-05-03 12:55:26 +00:00
|
|
|
|
{
|
|
|
|
|
lock (lifecycleLock)
|
|
|
|
|
{
|
|
|
|
|
return lifecycles[GetCurrentTestName()];
|
|
|
|
|
}
|
2023-04-26 12:40:54 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-04-12 14:06:04 +00:00
|
|
|
|
private void CreateNewTestLifecycle()
|
|
|
|
|
{
|
2023-05-03 12:18:37 +00:00
|
|
|
|
var testName = GetCurrentTestName();
|
2023-07-18 12:26:21 +00:00
|
|
|
|
fixtureLog.WriteLogTag();
|
2023-05-03 12:18:37 +00:00
|
|
|
|
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
|
2023-04-14 12:53:39 +00:00
|
|
|
|
{
|
2023-05-03 12:55:26 +00:00
|
|
|
|
lock (lifecycleLock)
|
|
|
|
|
{
|
2023-09-12 09:37:20 +00:00
|
|
|
|
var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString();
|
2023-09-04 07:08:34 +00:00
|
|
|
|
var lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration, GetTimeSet(), testNamespace);
|
2023-08-10 11:58:50 +00:00
|
|
|
|
lifecycles.Add(testName, lifecycle);
|
2023-09-04 07:08:34 +00:00
|
|
|
|
DefaultContainerRecipe.TestsType = TestsType;
|
2023-09-12 08:31:55 +00:00
|
|
|
|
//DefaultContainerRecipe.ApplicationIds = lifecycle.GetApplicationIds();
|
2023-05-03 12:55:26 +00:00
|
|
|
|
}
|
2023-04-14 12:53:39 +00:00
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void DisposeTestLifecycle()
|
|
|
|
|
{
|
2023-05-03 12:55:26 +00:00
|
|
|
|
var lifecycle = Get();
|
2023-07-21 07:20:28 +00:00
|
|
|
|
var testResult = GetTestResult();
|
|
|
|
|
var testDuration = lifecycle.GetTestDuration();
|
|
|
|
|
fixtureLog.Log($"{GetCurrentTestName()} = {testResult} ({testDuration})");
|
2023-09-12 08:31:55 +00:00
|
|
|
|
statusLog.ConcludeTest(testResult, testDuration);//, lifecycle.GetApplicationIds());
|
2023-04-14 12:53:39 +00:00
|
|
|
|
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
|
|
|
|
|
{
|
2023-09-12 08:31:55 +00:00
|
|
|
|
WriteEndTestLog(lifecycle.Log);
|
|
|
|
|
|
2023-05-03 12:55:26 +00:00
|
|
|
|
IncludeLogsAndMetricsOnTestFailure(lifecycle);
|
2023-04-14 12:53:39 +00:00
|
|
|
|
lifecycle.DeleteAllResources();
|
|
|
|
|
lifecycle = null!;
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2023-09-12 08:31:55 +00:00
|
|
|
|
private void WriteEndTestLog(TestLog log)
|
|
|
|
|
{
|
|
|
|
|
var result = TestContext.CurrentContext.Result;
|
|
|
|
|
|
|
|
|
|
Log($"*** Finished: {GetCurrentTestName()} = {result.Outcome.Status}");
|
|
|
|
|
if (!string.IsNullOrEmpty(result.Message))
|
|
|
|
|
{
|
|
|
|
|
Log(result.Message);
|
|
|
|
|
Log($"{result.StackTrace}");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
|
|
|
|
|
{
|
|
|
|
|
log.MarkAsFailed();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-05-04 06:55:20 +00:00
|
|
|
|
private ITimeSet GetTimeSet()
|
|
|
|
|
{
|
|
|
|
|
if (ShouldUseLongTimeouts()) return new LongTimeSet();
|
|
|
|
|
return new DefaultTimeSet();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private bool ShouldUseLongTimeouts()
|
|
|
|
|
{
|
|
|
|
|
// Don't be fooled! TestContext.CurrentTest.Test allows you easy access to the attributes of the current test.
|
|
|
|
|
// But this doesn't work for tests making use of [TestCase]. So instead, we use reflection here to figure out
|
|
|
|
|
// if the attribute is present.
|
|
|
|
|
var currentTest = TestContext.CurrentContext.Test;
|
|
|
|
|
var className = currentTest.ClassName;
|
|
|
|
|
var methodName = currentTest.MethodName;
|
|
|
|
|
|
|
|
|
|
var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray();
|
|
|
|
|
var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray();
|
|
|
|
|
|
|
|
|
|
return testMethods.Any(m => m.GetCustomAttribute<UseLongTimeoutsAttribute>() != null);
|
|
|
|
|
}
|
|
|
|
|
|
2023-05-03 12:55:26 +00:00
|
|
|
|
private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
|
2023-04-17 14:28:07 +00:00
|
|
|
|
{
|
|
|
|
|
var result = TestContext.CurrentContext.Result;
|
|
|
|
|
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
|
|
|
|
|
{
|
|
|
|
|
fixtureLog.MarkAsFailed();
|
|
|
|
|
|
|
|
|
|
if (IsDownloadingLogsAndMetricsEnabled())
|
|
|
|
|
{
|
|
|
|
|
lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
|
2023-09-11 09:59:33 +00:00
|
|
|
|
//DownloadAllLogs(lifecycle);
|
|
|
|
|
//DownloadAllMetrics(lifecycle);
|
2023-04-17 14:28:07 +00:00
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
lifecycle.Log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
//private void DownloadAllLogs(TestLifecycle lifecycle)
|
|
|
|
|
//{
|
|
|
|
|
// OnEachCodexNode(lifecycle, node =>
|
|
|
|
|
// {
|
|
|
|
|
// lifecycle.DownloadLog(node.CodexAccess.Container);
|
|
|
|
|
// });
|
|
|
|
|
//}
|
|
|
|
|
|
|
|
|
|
//private void DownloadAllMetrics(TestLifecycle lifecycle)
|
|
|
|
|
//{
|
|
|
|
|
// var metricsDownloader = new MetricsDownloader(lifecycle.Log);
|
|
|
|
|
|
|
|
|
|
// OnEachCodexNode(lifecycle, node =>
|
|
|
|
|
// {
|
|
|
|
|
// var m = node.Metrics as MetricsAccess;
|
|
|
|
|
// if (m != null)
|
|
|
|
|
// {
|
|
|
|
|
// metricsDownloader.DownloadAllMetricsForNode(node.GetName(), m);
|
|
|
|
|
// }
|
|
|
|
|
// });
|
|
|
|
|
//}
|
|
|
|
|
|
|
|
|
|
//private void OnEachCodexNode(TestLifecycle lifecycle, Action<OnlineCodexNode> action)
|
|
|
|
|
//{
|
|
|
|
|
// var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
|
|
|
|
|
// foreach (var node in allNodes)
|
|
|
|
|
// {
|
|
|
|
|
// action(node);
|
|
|
|
|
// }
|
|
|
|
|
//}
|
2023-04-12 14:06:04 +00:00
|
|
|
|
|
2023-04-14 12:53:39 +00:00
|
|
|
|
private string GetCurrentTestName()
|
|
|
|
|
{
|
|
|
|
|
return $"[{TestContext.CurrentContext.Test.Name}]";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private string GetTestResult()
|
|
|
|
|
{
|
|
|
|
|
return TestContext.CurrentContext.Result.Outcome.Status.ToString();
|
|
|
|
|
}
|
|
|
|
|
|
2023-04-13 09:53:54 +00:00
|
|
|
|
private bool IsDownloadingLogsAndMetricsEnabled()
|
|
|
|
|
{
|
|
|
|
|
var testProperties = TestContext.CurrentContext.Test.Properties;
|
|
|
|
|
return !testProperties.ContainsKey(DontDownloadLogsAndMetricsOnFailureAttribute.DontDownloadKey);
|
|
|
|
|
}
|
2023-04-12 14:06:04 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static class GlobalTestFailure
|
|
|
|
|
{
|
|
|
|
|
public static bool HasFailed { get; set; } = false;
|
|
|
|
|
}
|
|
|
|
|
}
|