diff --git a/ContinuousTests/Configuration.cs b/ContinuousTests/Configuration.cs index c4d5b7a..1d6b635 100644 --- a/ContinuousTests/Configuration.cs +++ b/ContinuousTests/Configuration.cs @@ -18,8 +18,8 @@ namespace ContinuousTests [Uniform("keep", "k", "KEEP", false, "Set to '1' to retain logs of successful tests.")] public bool KeepPassedTestLogs { get; set; } = false; - [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file.")] - public string KubeConfigFile { get; set; } = string.Empty; + [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")] + public string KubeConfigFile { get; set; } = "null"; public CodexDeployment CodexDeployment { get; set; } = null!; } @@ -30,7 +30,7 @@ namespace ContinuousTests { var uniformArgs = new ArgsUniform(args); - var result = uniformArgs.Parse(); + var result = uniformArgs.Parse(true); result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson); diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index 2d75c80..8d7fe44 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -20,15 +20,18 @@ namespace ContinuousTests startupChecker.Check(); var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview"); + overviewLog.Log("Continuous tests starting..."); var allTests = testFactory.CreateTests(); - var testStarters = allTests.Select(t => new TestStarter(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray(); + var testLoop = allTests.Select(t => new TestLoop(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray(); - foreach (var t in testStarters) + foreach (var t in testLoop) { + overviewLog.Log("Launching test-loop for " + t.Name); t.Begin(); Thread.Sleep(TimeSpan.FromMinutes(5)); } - + + overviewLog.Log("All test-loops launched."); while (true) Thread.Sleep((2 ^ 31) - 1); } } diff --git a/ContinuousTests/SingleTestRun.cs b/ContinuousTests/SingleTestRun.cs index 7e8d0a4..5d3d98d 100644 --- a/ContinuousTests/SingleTestRun.cs +++ b/ContinuousTests/SingleTestRun.cs @@ -4,6 +4,7 @@ using Logging; using Utils; using KubernetesWorkflow; using NUnit.Framework.Internal; +using System.Reflection; namespace ContinuousTests { @@ -41,20 +42,33 @@ namespace ContinuousTests try { RunTest(); - - if (!config.KeepPassedTestLogs) fixtureLog.Delete(); + fileManager.DeleteAllTestFiles(); + Directory.Delete(dataFolder, true); } catch (Exception ex) { - fixtureLog.Error("Test run failed with exception: " + ex); - fixtureLog.MarkAsFailed(); + overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex); + Environment.Exit(-1); } - fileManager.DeleteAllTestFiles(); - Directory.Delete(dataFolder, true); }); } private void RunTest() + { + try + { + RunTestMoments(); + + if (!config.KeepPassedTestLogs) fixtureLog.Delete(); + } + catch (Exception ex) + { + fixtureLog.Error("Test run failed with exception: " + ex); + fixtureLog.MarkAsFailed(); + } + } + + private void RunTestMoments() { var earliestMoment = handle.GetEarliestMoment(); @@ -66,7 +80,7 @@ namespace ContinuousTests if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any()) { Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping..."); - throw exceptions.Single(); + ThrowFailTest(); } var nextMoment = handle.GetNextMoment(t); @@ -80,9 +94,7 @@ namespace ContinuousTests { if (exceptions.Any()) { - var ex = exceptions.First(); - OverviewLog(" > Test failed: " + ex); - throw ex; + ThrowFailTest(); } OverviewLog(" > Test passed."); return; @@ -90,6 +102,28 @@ namespace ContinuousTests } } + private void ThrowFailTest() + { + var ex = UnpackException(exceptions.First()); + Log(ex.ToString()); + OverviewLog(" > Test failed: " + ex.Message); + throw ex; + } + + private Exception UnpackException(Exception exception) + { + if (exception is AggregateException a) + { + return UnpackException(a.InnerExceptions.First()); + } + if (exception is TargetInvocationException t) + { + return UnpackException(t.InnerException!); + } + + return exception; + } + private void RunMoment(int t) { using (var context = new TestExecutionContext.IsolatedContext()) @@ -100,7 +134,6 @@ namespace ContinuousTests } catch (Exception ex) { - Log($" > TestMoment yielded exception: " + ex); exceptions.Add(ex); } } diff --git a/ContinuousTests/TestStarter.cs b/ContinuousTests/TestLoop.cs similarity index 58% rename from ContinuousTests/TestStarter.cs rename to ContinuousTests/TestLoop.cs index 6253f64..8943735 100644 --- a/ContinuousTests/TestStarter.cs +++ b/ContinuousTests/TestLoop.cs @@ -2,29 +2,41 @@ namespace ContinuousTests { - public class TestStarter + public class TestLoop { private readonly Configuration config; private readonly BaseLog overviewLog; private readonly Type testType; private readonly TimeSpan runsEvery; - public TestStarter(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery) + public TestLoop(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery) { this.config = config; this.overviewLog = overviewLog; this.testType = testType; this.runsEvery = runsEvery; + + Name = testType.Name; } + public string Name { get; } + public void Begin() { Task.Run(() => { - while (true) + try { - StartTest(); - Thread.Sleep(runsEvery); + while (true) + { + StartTest(); + Thread.Sleep(runsEvery); + } + } + catch(Exception ex) + { + overviewLog.Error("Test infra failure: TestLoop failed with " + ex); + Environment.Exit(-1); } }); }