Debugging continuous test runner

ContinuousTests/Configuration.cs

@@ -18,8 +18,8 @@ namespace ContinuousTests
         [Uniform("keep", "k", "KEEP", false, "Set to '1' to retain logs of successful tests.")]
         public bool KeepPassedTestLogs { get; set; } = false;
 
-        [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file.")]
-        public string KubeConfigFile { get; set; } = string.Empty;
+        [Uniform("kube-config", "kc", "KUBECONFIG", true, "Path to Kubeconfig file. Use 'null' (default) to use local cluster.")]
+        public string KubeConfigFile { get; set; } = "null";
 
         public CodexDeployment CodexDeployment { get; set; } = null!;
     }
@@ -30,7 +30,7 @@ namespace ContinuousTests
         {
             var uniformArgs = new ArgsUniform<Configuration>(args);
 
-            var result = uniformArgs.Parse();
+            var result = uniformArgs.Parse(true);
             
             result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson);
 

ContinuousTests/ContinuousTestRunner.cs

@@ -20,15 +20,18 @@ namespace ContinuousTests
             startupChecker.Check();
 
             var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
+            overviewLog.Log("Continuous tests starting...");
             var allTests = testFactory.CreateTests();
-            var testStarters = allTests.Select(t => new TestStarter(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray();
+            var testLoop = allTests.Select(t => new TestLoop(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray();
 
-            foreach (var t in testStarters)
+            foreach (var t in testLoop)
             {
+                overviewLog.Log("Launching test-loop for " + t.Name);
                 t.Begin();
                 Thread.Sleep(TimeSpan.FromMinutes(5));
             }
 
+            overviewLog.Log("All test-loops launched.");
             while (true) Thread.Sleep((2 ^ 31) - 1);
         }
     }

ContinuousTests/SingleTestRun.cs

@@ -4,6 +4,7 @@ using Logging;
 using Utils;
 using KubernetesWorkflow;
 using NUnit.Framework.Internal;
+using System.Reflection;
 
 namespace ContinuousTests
 {
@@ -41,20 +42,33 @@ namespace ContinuousTests
                 try
                 {
                     RunTest();
-
-                    if (!config.KeepPassedTestLogs) fixtureLog.Delete();
+                    fileManager.DeleteAllTestFiles();
+                    Directory.Delete(dataFolder, true);
                 }
                 catch (Exception ex)
                 {
-                    fixtureLog.Error("Test run failed with exception: " + ex);
-                    fixtureLog.MarkAsFailed();
+                    overviewLog.Error("Test infra failure: SingleTestRun failed with " + ex);
+                    Environment.Exit(-1);
                 }
-                fileManager.DeleteAllTestFiles();
-                Directory.Delete(dataFolder, true);
             });
         }
 
         private void RunTest()
+        {
+            try
+            {
+                RunTestMoments();
+
+                if (!config.KeepPassedTestLogs) fixtureLog.Delete();
+            }
+            catch (Exception ex)
+            {
+                fixtureLog.Error("Test run failed with exception: " + ex);
+                fixtureLog.MarkAsFailed();
+            }
+        }
+
+        private void RunTestMoments()
         {
             var earliestMoment = handle.GetEarliestMoment();
 
@@ -66,7 +80,7 @@ namespace ContinuousTests
                 if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
                 {
                     Log("Exception detected. TestFailMode = StopAfterFirstFailure. Stopping...");
-                    throw exceptions.Single();
+                    ThrowFailTest();
                 }
 
                 var nextMoment = handle.GetNextMoment(t);
@@ -80,9 +94,7 @@ namespace ContinuousTests
                 {
                     if (exceptions.Any())
                     {
-                        var ex = exceptions.First();
-                        OverviewLog(" > Test failed: " + ex);
-                        throw ex;
+                        ThrowFailTest();
                     }
                     OverviewLog(" > Test passed.");
                     return;
@@ -90,6 +102,28 @@ namespace ContinuousTests
             }
         }
 
+        private void ThrowFailTest()
+        {
+            var ex = UnpackException(exceptions.First());
+            Log(ex.ToString());
+            OverviewLog(" > Test failed: " + ex.Message);
+            throw ex;
+        }
+
+        private Exception UnpackException(Exception exception)
+        {
+            if (exception is AggregateException a)
+            {
+                return UnpackException(a.InnerExceptions.First());
+            }
+            if (exception is TargetInvocationException t)
+            {
+                return UnpackException(t.InnerException!);
+            }
+
+            return exception;
+        }
+
         private void RunMoment(int t)
         {
             using (var context = new TestExecutionContext.IsolatedContext())
@@ -100,7 +134,6 @@ namespace ContinuousTests
                 }
                 catch (Exception ex)
                 {
-                    Log($" > TestMoment yielded exception: " + ex);
                     exceptions.Add(ex);
                 }
             }

ContinuousTests/TestLoop.cs

@@ -2,29 +2,41 @@
 
 namespace ContinuousTests
 {
-    public class TestStarter
+    public class TestLoop
     {
         private readonly Configuration config;
         private readonly BaseLog overviewLog;
         private readonly Type testType;
         private readonly TimeSpan runsEvery;
 
-        public TestStarter(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery)
+        public TestLoop(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery)
         {
             this.config = config;
             this.overviewLog = overviewLog;
             this.testType = testType;
             this.runsEvery = runsEvery;
+
+            Name = testType.Name;
         }
 
+        public string Name { get; }
+
         public void Begin()
         {
             Task.Run(() =>
             {
-                while (true)
+                try
                 {
-                    StartTest();
-                    Thread.Sleep(runsEvery);
+                    while (true)
+                    {
+                        StartTest();
+                        Thread.Sleep(runsEvery);
+                    }
+                }
+                catch(Exception ex)
+                {
+                    overviewLog.Error("Test infra failure: TestLoop failed with " + ex);
+                    Environment.Exit(-1);
                 }
             });
         }