vacp2p
/
cs-codex-dist-tests
mirror of https://github.com/vacp2p/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixes single-instance DistTest class being used to run multiple tests in parallel

This commit is contained in:
benbierens 2023-05-03 14:55:26 +02:00
parent 79a40904e4
commit 532eb3d4f9
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
4 changed files with 46 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
DistTestCore/DistTest.cs
View File

@ -6,7 +6,6 @@ using KubernetesWorkflow;
using Logging;
using NUnit.Framework;
using System.Reflection;
using Utils;
namespace DistTestCore
{
@ -15,14 +14,16 @@ namespace DistTestCore
{
private readonly Configuration configuration = new Configuration();
private readonly Assembly[] testAssemblies;
private FixtureLog fixtureLog = null!;
private TestLifecycle lifecycle = null!;
private DateTime testStart = DateTime.MinValue;
private readonly FixtureLog fixtureLog;
private readonly object lifecycleLock = new object();
private readonly Dictionary<string, TestLifecycle> lifecycles = new Dictionary<string, TestLifecycle>();
public DistTest()
{
var assemblies = AppDomain.CurrentDomain.GetAssemblies();
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
fixtureLog = new FixtureLog(configuration.GetLogConfig());
}
[OneTimeSetUp]
@ -31,7 +32,6 @@ namespace DistTestCore
// Previous test run may have been interrupted.
// Begin by cleaning everything up.
Timing.UseLongTimeouts = false;
fixtureLog = new FixtureLog(configuration.GetLogConfig());
try
{
@ -85,7 +85,7 @@ namespace DistTestCore
public TestFile GenerateTestFile(ByteSize size)
{
return lifecycle.FileManager.GenerateTestFile(size);
return Get().FileManager.GenerateTestFile(size);
}
public IOnlineCodexNode SetupCodexBootstrapNode()
@ -128,12 +128,20 @@ namespace DistTestCore
public ICodexNodeGroup BringOnline(ICodexSetup codexSetup)
{
return lifecycle.CodexStarter.BringOnline((CodexSetup)codexSetup);
return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
}
protected BaseLog Log
{
get { return lifecycle.Log; }
get { return Get().Log; }
}
private TestLifecycle Get()
{
lock (lifecycleLock)
{
return lifecycles[GetCurrentTestName()];
}
}
private bool ShouldUseLongTimeouts()
@ -156,24 +164,27 @@ namespace DistTestCore
var testName = GetCurrentTestName();
Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () =>
{
lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(), configuration);
testStart = DateTime.UtcNow;
lock (lifecycleLock)
{
lifecycles.Add(testName, new TestLifecycle(fixtureLog.CreateTestLog(), configuration));
}
});
}
private void DisposeTestLifecycle()
{
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({GetTestDuration()})");
var lifecycle = Get();
fixtureLog.Log($"{GetCurrentTestName()} = {GetTestResult()} ({lifecycle.GetTestDuration()})");
Stopwatch.Measure(fixtureLog, $"Teardown for {GetCurrentTestName()}", () =>
{
lifecycle.Log.EndTest();
IncludeLogsAndMetricsOnTestFailure();
IncludeLogsAndMetricsOnTestFailure(lifecycle);
lifecycle.DeleteAllResources();
lifecycle = null!;
});
}
private void IncludeLogsAndMetricsOnTestFailure()
private void IncludeLogsAndMetricsOnTestFailure(TestLifecycle lifecycle)
{
var result = TestContext.CurrentContext.Result;
if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
@ -183,8 +194,8 @@ namespace DistTestCore
if (IsDownloadingLogsAndMetricsEnabled())
{
lifecycle.Log.Log("Downloading all CodexNode logs and metrics because of test failure...");
DownloadAllLogs();
DownloadAllMetrics();
DownloadAllLogs(lifecycle);
DownloadAllMetrics(lifecycle);
}
else
{
@ -193,25 +204,19 @@ namespace DistTestCore
}
}
private string GetTestDuration()
private void DownloadAllLogs(TestLifecycle lifecycle)
{
var testDuration = DateTime.UtcNow - testStart;
return Time.FormatDuration(testDuration);
}
private void DownloadAllLogs()
{
OnEachCodexNode(node =>
OnEachCodexNode(lifecycle, node =>
{
lifecycle.DownloadLog(node);
});
}
private void DownloadAllMetrics()
private void DownloadAllMetrics(TestLifecycle lifecycle)
{
var metricsDownloader = new MetricsDownloader(lifecycle.Log);
OnEachCodexNode(node =>
OnEachCodexNode(lifecycle, node =>
{
var m = node.Metrics as MetricsAccess;
if (m != null)
@ -221,7 +226,7 @@ namespace DistTestCore
});
}
private void OnEachCodexNode(Action<OnlineCodexNode> action)
private void OnEachCodexNode(TestLifecycle lifecycle, Action<OnlineCodexNode> action)
{
var allNodes = lifecycle.CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
foreach (var node in allNodes)

4
DistTestCore/FileManager.cs
View File

@ -1,5 +1,6 @@
using Logging;
using NUnit.Framework;
using Utils;
namespace DistTestCore
{
@ -13,13 +14,14 @@ namespace DistTestCore
public class FileManager : IFileManager
{
public const int ChunkSize = 1024 * 1024;
private static NumberSource folderNumberSource = new NumberSource(0);
private readonly Random random = new Random();
private readonly TestLog log;
private readonly string folder;
public FileManager(TestLog log, Configuration configuration)
{
folder = configuration.GetFileManagerFolder();
folder = Path.Combine(configuration.GetFileManagerFolder(), folderNumberSource.GetNextNumber().ToString("D5"));
EnsureDirectory();
this.log = log;

9
DistTestCore/TestLifecycle.cs
View File

@ -1,12 +1,14 @@
using DistTestCore.Logs;
using KubernetesWorkflow;
using Logging;
using Utils;
namespace DistTestCore
{
public class TestLifecycle
{
private readonly WorkflowCreator workflowCreator;
private DateTime testStart = DateTime.MinValue;
public TestLifecycle(TestLog log, Configuration configuration)
{
@ -17,6 +19,7 @@ namespace DistTestCore
CodexStarter = new CodexStarter(this, workflowCreator);
PrometheusStarter = new PrometheusStarter(this, workflowCreator);
GethStarter = new GethStarter(this, workflowCreator);
testStart = DateTime.UtcNow;
}
public TestLog Log { get; }
@ -42,5 +45,11 @@ namespace DistTestCore
return new CodexNodeLog(subFile, node);
}
public string GetTestDuration()
{
var testDuration = DateTime.UtcNow - testStart;
return Time.FormatDuration(testDuration);
}
}
}

4
KubernetesWorkflow/WorkflowCreator.cs
View File

@ -10,11 +10,13 @@ namespace KubernetesWorkflow
private readonly KnownK8sPods knownPods = new KnownK8sPods();
private readonly K8sCluster cluster;
private readonly BaseLog log;
private readonly string testNamespace;
public WorkflowCreator(BaseLog log, Configuration configuration)
{
cluster = new K8sCluster(configuration);
this.log = log;
testNamespace = ApplicationLifecycle.Instance.GetTestNamespace();
}
public StartupWorkflow CreateWorkflow()
@ -23,7 +25,7 @@ namespace KubernetesWorkflow
ApplicationLifecycle.Instance.GetServiceNumberSource(),
containerNumberSource);
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, ApplicationLifecycle.Instance.GetTestNamespace());
return new StartupWorkflow(log, workflowNumberSource, cluster, knownPods, testNamespace);
}
}
}