cs-codex-dist-tests/ContinuousTests/SingleTestRun.cs

254 lines
8.7 KiB
C#
Raw Normal View History

using DistTestCore.Codex;
using DistTestCore;
using Logging;
using Utils;
2023-06-23 08:14:16 +00:00
using KubernetesWorkflow;
using NUnit.Framework.Internal;
2023-06-27 08:16:59 +00:00
using System.Reflection;
using static Program;
using FileUtils;
namespace ContinuousTests
{
public class SingleTestRun
{
2023-06-29 14:07:49 +00:00
private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
2023-06-25 07:53:10 +00:00
private readonly List<Exception> exceptions = new List<Exception>();
2023-06-28 14:19:37 +00:00
private readonly TaskFactory taskFactory;
private readonly Configuration config;
2023-06-25 09:06:47 +00:00
private readonly BaseLog overviewLog;
2023-06-25 07:53:10 +00:00
private readonly TestHandle handle;
2023-06-28 14:19:37 +00:00
private readonly CancellationToken cancelToken;
2023-06-29 14:07:49 +00:00
private readonly CodexAccess[] nodes;
private readonly FileManager fileManager;
2023-06-25 07:53:10 +00:00
private readonly FixtureLog fixtureLog;
2023-06-25 09:24:32 +00:00
private readonly string testName;
2023-06-25 08:50:01 +00:00
private readonly string dataFolder;
private static int failureCount = 0;
public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, StartupChecker startupChecker, CancellationToken cancelToken)
{
2023-06-28 14:19:37 +00:00
this.taskFactory = taskFactory;
this.config = config;
2023-06-25 09:06:47 +00:00
this.overviewLog = overviewLog;
2023-06-25 07:53:10 +00:00
this.handle = handle;
2023-06-28 14:19:37 +00:00
this.cancelToken = cancelToken;
2023-06-25 09:24:32 +00:00
testName = handle.Test.GetType().Name;
2023-07-18 07:47:44 +00:00
fixtureLog = new FixtureLog(new LogConfig(config.LogPath, true), DateTime.UtcNow, testName);
ApplyLogReplacements(fixtureLog, startupChecker);
nodes = CreateRandomNodes();
2023-06-25 08:50:01 +00:00
dataFolder = config.DataPath + "-" + Guid.NewGuid();
fileManager = new FileManager(fixtureLog, CreateFileManagerConfiguration().GetFileManagerFolder());
}
public void Run(EventWaitHandle runFinishedHandle)
{
2023-06-28 14:19:37 +00:00
taskFactory.Run(() =>
2023-06-25 07:53:10 +00:00
{
try
{
RunTest();
2023-06-27 08:16:59 +00:00
fileManager.DeleteAllTestFiles();
Directory.Delete(dataFolder, true);
runFinishedHandle.Set();
2023-06-25 07:53:10 +00:00
}
catch (Exception ex)
{
2023-06-27 08:16:59 +00:00
overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex);
Environment.Exit(-1);
2023-06-25 07:53:10 +00:00
}
});
}
private void RunTest()
2023-06-27 08:16:59 +00:00
{
try
{
RunTestMoments();
if (!config.KeepPassedTestLogs) fixtureLog.Delete();
}
catch (Exception ex)
{
fixtureLog.Error("Test run failed with exception: " + ex);
fixtureLog.MarkAsFailed();
failureCount++;
if (config.StopOnFailure > 0)
{
OverviewLog($"Failures: {failureCount} / {config.StopOnFailure}");
if (failureCount >= config.StopOnFailure)
{
OverviewLog($"Configured to stop after {config.StopOnFailure} failures. Downloading cluster logs...");
DownloadClusterLogs();
OverviewLog("Log download finished. Cancelling test runner...");
Cancellation.Cts.Cancel();
}
}
2023-06-27 08:16:59 +00:00
}
}
private void ApplyLogReplacements(FixtureLog fixtureLog, StartupChecker startupChecker)
{
foreach (var replacement in startupChecker.LogReplacements) fixtureLog.AddStringReplace(replacement.From, replacement.To);
}
2023-06-27 08:16:59 +00:00
private void RunTestMoments()
2023-06-25 07:53:10 +00:00
{
var earliestMoment = handle.GetEarliestMoment();
var t = earliestMoment;
2023-06-25 08:50:01 +00:00
while (true)
2023-06-25 07:53:10 +00:00
{
2023-06-28 14:19:37 +00:00
cancelToken.ThrowIfCancellationRequested();
2023-06-25 07:53:10 +00:00
RunMoment(t);
if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
{
Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping...");
2023-06-27 08:16:59 +00:00
ThrowFailTest();
2023-06-25 07:53:10 +00:00
}
var nextMoment = handle.GetNextMoment(t);
if (nextMoment != null)
{
2023-06-28 14:19:37 +00:00
var delta = TimeSpan.FromSeconds(nextMoment.Value - t);
Log($" > Next TestMoment in {Time.FormatDuration(delta)} seconds...");
cancelToken.WaitHandle.WaitOne(delta);
t = nextMoment.Value;
2023-06-25 07:53:10 +00:00
}
else
{
2023-06-25 08:50:01 +00:00
if (exceptions.Any())
{
2023-06-27 08:16:59 +00:00
ThrowFailTest();
2023-06-25 08:50:01 +00:00
}
2023-09-01 06:45:23 +00:00
OverviewLog(" > Test passed.");
2023-06-25 08:50:01 +00:00
return;
2023-06-25 07:53:10 +00:00
}
}
}
2023-06-27 08:16:59 +00:00
private void ThrowFailTest()
{
var exs = UnpackExceptions(exceptions);
var exceptionsMessage = GetCombinedExceptionsMessage(exs);
Log(exceptionsMessage);
2023-09-01 06:45:23 +00:00
OverviewLog($" > Test failed: " + exceptionsMessage);
throw new Exception(exceptionsMessage);
2023-06-27 08:16:59 +00:00
}
private void DownloadClusterLogs()
{
var k8sFactory = new K8sFactory();
2023-09-01 07:28:04 +00:00
var log = new NullLog();
log.FullFilename = Path.Combine(config.LogPath, "NODE");
var lifecycle = k8sFactory.CreateTestLifecycle(config.KubeConfigFile, config.LogPath, "dataPath", config.CodexDeployment.Metadata.KubeNamespace, new DefaultTimeSet(), log);
foreach (var container in config.CodexDeployment.CodexContainers)
{
lifecycle.DownloadLog(container);
}
}
private string GetCombinedExceptionsMessage(Exception[] exceptions)
{
return string.Join(Environment.NewLine, exceptions.Select(ex => ex.ToString()));
}
private Exception[] UnpackExceptions(List<Exception> exceptions)
{
return exceptions.Select(UnpackException).ToArray();
}
2023-06-27 08:16:59 +00:00
private Exception UnpackException(Exception exception)
{
if (exception is AggregateException a)
{
return UnpackException(a.InnerExceptions.First());
}
if (exception is TargetInvocationException t)
{
return UnpackException(t.InnerException!);
}
return exception;
}
2023-06-25 07:53:10 +00:00
private void RunMoment(int t)
{
using (var context = new TestExecutionContext.IsolatedContext())
2023-06-25 07:53:10 +00:00
{
try
{
handle.InvokeMoment(t, InitializeTest);
}
catch (Exception ex)
{
exceptions.Add(ex);
}
2023-06-25 07:53:10 +00:00
}
DecommissionTest();
}
private void InitializeTest(string name)
{
Log($" > Running TestMoment '{name}'");
2023-06-28 14:19:37 +00:00
handle.Test.Initialize(nodes, fixtureLog, fileManager, config, cancelToken);
2023-06-25 07:53:10 +00:00
}
private void DecommissionTest()
{
2023-06-28 14:19:37 +00:00
handle.Test.Initialize(null!, null!, null!, null!, cancelToken);
}
2023-06-25 07:53:10 +00:00
private void Log(string msg)
{
2023-06-25 07:53:10 +00:00
fixtureLog.Log(msg);
}
2023-06-25 09:06:47 +00:00
private void OverviewLog(string msg)
{
Log(msg);
var containerNames = GetContainerNames();
overviewLog.Log($"{containerNames} {testName}: {msg}");
2023-06-25 09:06:47 +00:00
}
private string GetContainerNames()
{
if (handle.Test.RequiredNumberOfNodes == -1) return "(All Nodes)";
return $"({string.Join(",", nodes.Select(n => n.Container.Name))})";
}
private CodexAccess[] CreateRandomNodes()
{
var containers = SelectRandomContainers();
2023-06-25 07:53:10 +00:00
fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));
return codexNodeFactory.Create(config, containers, fixtureLog, handle.Test.TimeSet);
}
private RunningContainer[] SelectRandomContainers()
{
var number = handle.Test.RequiredNumberOfNodes;
if (number == -1) return config.CodexDeployment.CodexContainers;
2023-06-23 08:14:16 +00:00
var containers = config.CodexDeployment.CodexContainers.ToList();
var result = new RunningContainer[number];
for (var i = 0; i < number; i++)
{
2023-06-23 08:14:16 +00:00
result[i] = containers.PickOneRandom();
}
return result;
}
2023-06-25 07:53:10 +00:00
private DistTestCore.Configuration CreateFileManagerConfiguration()
{
2023-06-25 08:50:01 +00:00
return new DistTestCore.Configuration(null, string.Empty, false, dataFolder,
CodexLogLevel.Error, string.Empty);
2023-06-25 07:53:10 +00:00
}
}
}