Setting up cancelling

ContinuousTests/ContinuousTest.cs

@@ -21,12 +21,13 @@ namespace ContinuousTests
 
         private const string UploadFailedMessage = "Unable to store block";
 
-        public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager, Configuration configuration)
+        public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager, Configuration configuration, CancellationToken cancelToken)
         {
             Nodes = nodes;
             Log = log;
             FileManager = fileManager;
             Configuration = configuration;
+            CancelToken = cancelToken;
 
             if (nodes != null)
             {
@@ -43,6 +44,7 @@ namespace ContinuousTests
         public IFileManager FileManager { get; private set; } = null!;
         public Configuration Configuration { get; private set; } = null!;
         public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } }
+        public CancellationToken CancelToken { get; private set; } = null;
         public NodeRunner NodeRunner { get; private set; } = null!;
 
         public abstract int RequiredNumberOfNodes { get; }

ContinuousTests/ContinuousTestRunner.cs

@@ -10,34 +10,41 @@ namespace ContinuousTests
         private readonly TestFactory testFactory = new TestFactory();
         private readonly Configuration config;
         private readonly StartupChecker startupChecker;
+        private readonly CancellationToken cancelToken;
 
-        public ContinuousTestRunner(string[] args)
+        public ContinuousTestRunner(string[] args, CancellationToken cancelToken)
         {
             config = configLoader.Load(args);
             startupChecker = new StartupChecker(config);
+            this.cancelToken = cancelToken;
         }
 
         public void Run()
         {
             startupChecker.Check();
 
+            var taskFactory = new TaskFactory();
             var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
             overviewLog.Log("Continuous tests starting...");
             var allTests = testFactory.CreateTests();
 
             ClearAllCustomNamespaces(allTests, overviewLog);
 
-            var testLoop = allTests.Select(t => new TestLoop(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray();
+            var testLoops = allTests.Select(t => new TestLoop(taskFactory, config, overviewLog, t.GetType(), t.RunTestEvery, cancelToken)).ToArray();
 
-            foreach (var t in testLoop)
+            foreach (var testLoop in testLoops)
             {
-                overviewLog.Log("Launching test-loop for " + t.Name);
+                cancelToken.ThrowIfCancellationRequested();
-                t.Begin();
+
+                overviewLog.Log("Launching test-loop for " + testLoop.Name);
+                testLoop.Begin();
                 Thread.Sleep(TimeSpan.FromSeconds(15));
             }
 
             overviewLog.Log("All test-loops launched.");
-            while (true) Thread.Sleep((2 ^ 31) - 1);
+            cancelToken.WaitHandle.WaitOne();
+            overviewLog.Log("Cancelling all test-loops...");
+            taskFactory.WaitAll();
         }
 
         private void ClearAllCustomNamespaces(ContinuousTest[] allTests, FixtureLog log)

ContinuousTests/Program.cs

@@ -6,7 +6,19 @@ public class Program
     {
         Console.WriteLine("Codex Continous-Test-Runner.");
         Console.WriteLine("Running...");
-        var runner = new ContinuousTestRunner(args);
+
+        var cts = new CancellationTokenSource();
+        var runner = new ContinuousTestRunner(args, cts.Token);
+
+        Console.CancelKeyPress += (sender, e) =>
+        {
+            Console.WriteLine("Stopping...");
+            e.Cancel = true;
+
+            cts.Cancel();
+        };
+        
         runner.Run();
+        Console.WriteLine("Done.");
     }
 }

ContinuousTests/SingleTestRun.cs

@@ -12,21 +12,24 @@ namespace ContinuousTests
     {
         private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
         private readonly List<Exception> exceptions = new List<Exception>();
+        private readonly TaskFactory taskFactory;
         private readonly Configuration config;
         private readonly BaseLog overviewLog;
         private readonly TestHandle handle;
+        private readonly CancellationToken cancelToken;
         private readonly CodexNode[] nodes;
         private readonly FileManager fileManager;
         private readonly FixtureLog fixtureLog;
         private readonly string testName;
         private readonly string dataFolder;
 
-        public SingleTestRun(Configuration config, BaseLog overviewLog, TestHandle handle)
+        public SingleTestRun(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, TestHandle handle, CancellationToken cancelToken)
         {
+            this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
             this.handle = handle;
-
+            this.cancelToken = cancelToken;
             testName = handle.Test.GetType().Name;
             fixtureLog = new FixtureLog(new LogConfig(config.LogPath, false), testName);
 
@@ -37,7 +40,7 @@ namespace ContinuousTests
 
         public void Run()
         {
-            Task.Run(() =>
+            taskFactory.Run(() =>
             {
                 try
                 {
@@ -75,6 +78,8 @@ namespace ContinuousTests
             var t = earliestMoment;
             while (true)
             {
+                cancelToken.ThrowIfCancellationRequested();
+
                 RunMoment(t);
 
                 if (handle.Test.TestFailMode == TestFailMode.StopAfterFirstFailure && exceptions.Any())
@@ -86,9 +91,10 @@ namespace ContinuousTests
                 var nextMoment = handle.GetNextMoment(t);
                 if (nextMoment != null)
                 {
-                    Log($" > Next TestMoment in {nextMoment.Value} seconds...");
+                    var delta = TimeSpan.FromSeconds(nextMoment.Value - t);
-                    t += nextMoment.Value;
+                    Log($" > Next TestMoment in {Time.FormatDuration(delta)} seconds...");
-                    Thread.Sleep(nextMoment.Value * 1000);
+                    cancelToken.WaitHandle.WaitOne(delta);
+                    t = nextMoment.Value;
                 }
                 else
                 {
@@ -144,12 +150,12 @@ namespace ContinuousTests
         private void InitializeTest(string name)
         {
             Log($" > Running TestMoment '{name}'");
-            handle.Test.Initialize(nodes, fixtureLog, fileManager, config);
+            handle.Test.Initialize(nodes, fixtureLog, fileManager, config, cancelToken);
         }
 
         private void DecommissionTest()
         {
-            handle.Test.Initialize(null!, null!, null!, null!);
+            handle.Test.Initialize(null!, null!, null!, null!, cancelToken);
         }
 
         private void Log(string msg)

ContinuousTests/TaskFactory.cs

@@ -0,0 +1,39 @@
+namespace ContinuousTests
+{
+    public class TaskFactory
+    {
+        private readonly object taskLock = new();
+        private readonly List<Task> activeTasks = new List<Task>();
+
+        public void Run(Action action)
+        {
+            lock (taskLock)
+            {
+                activeTasks.Add(Task.Run(action).ContinueWith(CleanupTask, null));
+            }
+        }
+
+        public void WaitAll()
+        {
+            var tasks = activeTasks.ToArray();
+            Task.WaitAll(tasks);
+
+            var moreTasks = false;
+            lock (taskLock)
+            {
+                activeTasks.RemoveAll(task => task.IsCompleted);
+                moreTasks = activeTasks.Any();
+            }
+
+            if (moreTasks) WaitAll();
+        }
+
+        private void CleanupTask(Task completedTask, object? arg)
+        {
+            lock (taskLock)
+            {
+                activeTasks.Remove(completedTask);
+            }
+        }
+    }
+}

ContinuousTests/TestLoop.cs

@@ -4,18 +4,21 @@ namespace ContinuousTests
 {
     public class TestLoop
     {
+        private readonly TaskFactory taskFactory;
         private readonly Configuration config;
         private readonly BaseLog overviewLog;
         private readonly Type testType;
         private readonly TimeSpan runsEvery;
+        private readonly CancellationToken cancelToken;
 
-        public TestLoop(Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery)
+        public TestLoop(TaskFactory taskFactory, Configuration config, BaseLog overviewLog, Type testType, TimeSpan runsEvery, CancellationToken cancelToken)
         {
+            this.taskFactory = taskFactory;
             this.config = config;
             this.overviewLog = overviewLog;
             this.testType = testType;
             this.runsEvery = runsEvery;
-
+            this.cancelToken = cancelToken;
             Name = testType.Name;
         }
 
@@ -23,17 +26,23 @@ namespace ContinuousTests
 
         public void Begin()
         {
-            Task.Run(() =>
+            taskFactory.Run(() =>
             {
                 try
                 {
                     while (true)
                     {
+                        cancelToken.ThrowIfCancellationRequested();
                         StartTest();
-                        Thread.Sleep(runsEvery);
+
+                        cancelToken.WaitHandle.WaitOne(runsEvery);
                     }
                 }
-                catch(Exception ex)
+                catch (OperationCanceledException)
+                {
+                    overviewLog.Log("Test-loop " + testType.Name + " is cancelled.");
+                }
+                catch (Exception ex)
                 {
                     overviewLog.Error("Test infra failure: TestLoop failed with " + ex);
                     Environment.Exit(-1);
@@ -45,7 +54,7 @@ namespace ContinuousTests
         {
             var test = (ContinuousTest)Activator.CreateInstance(testType)!;
             var handle = new TestHandle(test);
-            var run = new SingleTestRun(config, overviewLog, handle);
+            var run = new SingleTestRun(taskFactory, config, overviewLog, handle, cancelToken);
             run.Run();
         }
     }