diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index b0686e15..175fae60 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -6,7 +6,6 @@ using KubernetesWorkflow; using Logging; using NUnit.Framework; using System.Reflection; -using Utils; namespace DistTestCore { @@ -15,14 +14,16 @@ namespace DistTestCore { private readonly Configuration configuration = new Configuration(); private readonly Assembly[] testAssemblies; - private FixtureLog fixtureLog = null!; - private TestLifecycle lifecycle = null!; - private DateTime testStart = DateTime.MinValue; + private readonly FixtureLog fixtureLog; + private readonly object lifecycleLock = new object(); + private readonly Dictionary lifecycles = new Dictionary(); public DistTest() { var assemblies = AppDomain.CurrentDomain.GetAssemblies(); testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray(); + + fixtureLog = new FixtureLog(configuration.GetLogConfig()); } [OneTimeSetUp] @@ -31,7 +32,6 @@ namespace DistTestCore // Previous test run may have been interrupted. // Begin by cleaning everything up. Timing.UseLongTimeouts = false; - fixtureLog = new FixtureLog(configuration.GetLogConfig()); try { @@ -85,7 +85,7 @@ namespace DistTestCore public TestFile GenerateTestFile(ByteSize size) { - return lifecycle.FileManager.GenerateTestFile(size); + return Get().FileManager.GenerateTestFile(size); } public IOnlineCodexNode SetupCodexBootstrapNode() @@ -128,12 +128,20 @@ namespace DistTestCore public ICodexNodeGroup BringOnline(ICodexSetup codexSetup) { - return lifecycle.CodexStarter.BringOnline((CodexSetup)codexSetup); + return Get().CodexStarter.BringOnline((CodexSetup)codexSetup); } protected BaseLog Log { - get { return lifecycle.Log; } + get { return Get().Log; } + } + + private TestLifecycle Get() + { + lock (lifecycleLock) + { + return lifecycles[GetCurrentTestName()]; + } } private bool ShouldUseLongTimeouts() @@ -156,24 +164,27 @@ namespace DistTestCore var testName = GetCurrentTestName(); Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () => { - lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration); - testStart = DateTime.UtcNow; + lock (lifecycleLock) + { + lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration)); + } }); } private void DisposeTestLifecycle() { - fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({GetTestDuration()})"); + var lifecycle = Get(); + fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})"); Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () => { lifecycle.Log.EndTest(); - IncludeLogsAndMetricsOnTestFailure(); + IncludeLogsAndMetricsOnTestFailure(lifecycle); lifecycle.DeleteAllResources(); lifecycle = null!; }); } - private void IncludeLogsAndMetricsOnTestFailure() + private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle) { var result = TestContext.CurrentContext.Result; if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) @@ -183,8 +194,8 @@ namespace DistTestCore if (IsDownloadingLogsAndMetricsEnabled()) { lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure..."); - DownloadAllLogs(); - DownloadAllMetrics(); + DownloadAllLogs(lifecycle); + DownloadAllMetrics(lifecycle); } else { @@ -193,25 +204,19 @@ namespace DistTestCore } } - private string GetTestDuration() + private void DownloadAllLogs(TestLifecycle lifecycle) { - var testDuration = DateTime.UtcNow - testStart; - return Time.FormatDuration(testDuration); - } - - private void DownloadAllLogs() - { - OnEachCodexNode(node => + OnEachCodexNode(lifecycle, node => { lifecycle.DownloadLog(node); }); } - private void DownloadAllMetrics() + private void DownloadAllMetrics(TestLifecycle lifecycle) { var metricsDownloader = new MetricsDownloader(lifecycle.Log); - OnEachCodexNode(node => + OnEachCodexNode(lifecycle, node => { var m = node.Metrics as MetricsAccess; if (m != null) @@ -221,7 +226,7 @@ namespace DistTestCore }); } - private void OnEachCodexNode(Action action) + private void OnEachCodexNode(TestLifecycle lifecycle, Action action) { var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes); foreach (var node in allNodes) diff --git a/DistTestCore/FileManager.cs b/DistTestCore/FileManager.cs index b195e9c0..ae58cd64 100644 --- a/DistTestCore/FileManager.cs +++ b/DistTestCore/FileManager.cs @@ -1,5 +1,6 @@ using Logging; using NUnit.Framework; +using Utils; namespace DistTestCore { @@ -13,13 +14,14 @@ namespace DistTestCore public class FileManager : IFileManager { public const int ChunkSize = 1024 * 1024; + private static NumberSource folderNumberSource = new NumberSource(0); private readonly Random random = new Random(); private readonly TestLog log; private readonly string folder; public FileManager(TestLog log, Configuration configuration) { - folder = configuration.GetFileManagerFolder(); + folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5")); EnsureDirectory(); this.log = log; diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 4dddf34b..09974e9f 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,12 +1,14 @@ using DistTestCore.Logs; using KubernetesWorkflow; using Logging; +using Utils; namespace DistTestCore { public class TestLifecycle { private readonly WorkflowCreator workflowCreator; + private DateTime testStart = DateTime.MinValue; public TestLifecycle(TestLog log, Configuration configuration) { @@ -17,6 +19,7 @@ namespace DistTestCore CodexStarter = new CodexStarter(this, workflowCreator); PrometheusStarter = new PrometheusStarter(this, workflowCreator); GethStarter = new GethStarter(this, workflowCreator); + testStart = DateTime.UtcNow; } public TestLog Log { get; } @@ -42,5 +45,11 @@ namespace DistTestCore return new CodexNodeLog(subFile, node); } + + public string GetTestDuration() + { + var testDuration = DateTime.UtcNow - testStart; + return Time.FormatDuration(testDuration); + } } } diff --git a/KubernetesWorkflow/WorkflowCreator.cs b/KubernetesWorkflow/WorkflowCreator.cs index e4f8032c..aa0a098c 100644 --- a/KubernetesWorkflow/WorkflowCreator.cs +++ b/KubernetesWorkflow/WorkflowCreator.cs @@ -10,11 +10,13 @@ namespace KubernetesWorkflow private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly K8sCluster cluster; private readonly BaseLog log; + private readonly string testNamespace; public WorkflowCreator(BaseLog log, Configuration configuration) { cluster = new K8sCluster(configuration); this.log = log; + testNamespace = ApplicationLifecycle.Instance.GetTestNamespace(); } public StartupWorkflow CreateWorkflow() @@ -23,7 +25,7 @@ namespace KubernetesWorkflow ApplicationLifecycle.Instance.GetServiceNumberSource(), containerNumberSource); - return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, ApplicationLifecycle.Instance.GetTestNamespace()); + return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace); } } }