Better logging alignment for continous tests

ContinuousTests/AllTestsRun.cs

@@ -1,64 +1,86 @@
 using DistTestCore;
 using DistTestCore.Codex;
 using Logging;
+using Utils;
 
 namespace ContinuousTests
 {
-    public class TestRun
+    public class AllTestsRun
     {
-        private readonly Random random = new Random();
-        private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
         private readonly Configuration config;
-        private readonly BaseLog log;
+        private readonly FixtureLog log;
         private readonly TestFactory testFinder;
-        private readonly FileManager fileManager;
-        private ITimeSet timeSet;
 
-        public TestRun(Configuration config, BaseLog log, TestFactory testFinder)
+        public AllTestsRun(Configuration config, FixtureLog log, TestFactory testFinder)
         {
             this.config = config;
             this.log = log;
             this.testFinder = testFinder;
-            fileManager = new FileManager(log, new DistTestCore.Configuration());
-            timeSet = new DefaultTimeSet();
         }
 
-        public void Run()
+        public ContinuousTestResult RunAll()
         {
             var remainingTests = testFinder.CreateTests().ToList();
+            var result = ContinuousTestResult.Passed;
             while (remainingTests.Any())
             {
-                var test = PickOneRandom(remainingTests);
+                var test = remainingTests.PickOneRandom();
-                var nodes = CreateRandomNodes(test.RequiredNumberOfNodes);
+                var testLog = log.CreateTestLog(test.Name);
-                AssignEssentials(test, nodes);
+                var singleTestRun = new SingleTestRun(config, test, testLog);
-                fileManager.PushFileSet();
 
                 log.Log($"Start '{test.Name}'");
                 try
                 {
-                    test.Run();
+                    singleTestRun.Run();
                     log.Log($"'{test.Name}' = Passed");
                 }
                 catch
                 {
                     log.Log($"'{test.Name}' = Failed");
+                    testLog.MarkAsFailed();
+                    result = ContinuousTestResult.Failed;
                 }
                 
-                fileManager.PopFileSet();
-                ClearEssentials(test);
                 Thread.Sleep(config.SleepSecondsPerTest * 1000);
             }
+            return result;
+        }
+    }
+
+    public class SingleTestRun
+    {
+        private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
+        private readonly Configuration config;
+        private readonly ContinuousTest test;
+        private readonly CodexNode[] nodes;
+        private readonly FileManager fileManager;
+
+        public SingleTestRun(Configuration config, ContinuousTest test, BaseLog testLog)
+        {
+            this.config = config;
+            this.test = test;
+
+            nodes = CreateRandomNodes(test.RequiredNumberOfNodes, testLog);
+            fileManager = new FileManager(testLog, new DistTestCore.Configuration());
+
+            test.Initialize(nodes, testLog, fileManager);
         }
 
-        private void AssignEssentials(ContinuousTest test, CodexNode[] nodes)
+        public void Run()
         {
-            test.Initialize(nodes, log, fileManager);
+            test.Run();
         }
 
-        private void ClearEssentials(ContinuousTest test)
+        public void TearDown()
         {
-            // Looks a little strange, but prevents finished test from interacting further.
             test.Initialize(null!, null!, null!);
+            fileManager.DeleteAllTestFiles();
+        }
+
+        private CodexNode[] CreateRandomNodes(int number, BaseLog testLog)
+        {
+            var urls = SelectRandomUrls(number);
+            return codexNodeFactory.Create(urls, testLog, test.TimeSet);
         }
 
         private string[] SelectRandomUrls(int number)
@@ -67,23 +89,9 @@ namespace ContinuousTests
             var result = new string[number];
             for (var i = 0; i < number; i++)
             {
-                result[i] = PickOneRandom(urls);
+                result[i] = urls.PickOneRandom();
             }
             return result;
         }
-
-        private CodexNode[] CreateRandomNodes(int number)
-        {
-            var urls = SelectRandomUrls(number);
-            return codexNodeFactory.Create(urls, log, timeSet);
-        }
-
-        private T PickOneRandom<T>(List<T> remainingItems)
-        {
-            var i = random.Next(0, remainingItems.Count);
-            var result = remainingItems[i];
-            remainingItems.RemoveAt(i);
-            return result;
-        }
     }
 }

ContinuousTests/Configuration.cs

@@ -16,6 +16,10 @@ namespace ContinuousTests
         public Configuration Load()
         {
             var config = Read();
+            //config.LogPath = "logs";
+            //config.SleepSecondsPerTest = 10;
+            //config.CodexUrls = new string[] { "http://localhost:8080", "http://localhost:8081" };
+
             Validate(config);
             return config;
         }

ContinuousTests/ContinuousTestRunner.cs

@@ -13,33 +13,43 @@ namespace ContinuousTests
         public void Run()
         {
             var config = configLoader.Load();
-            var log = new TestLog(config.LogPath, true);
+            StartupChecks(config);
-
-            log.Log("Starting continuous test run...");
-            log.Log("Checking configuration...");
-            PreflightCheck(config);
-            log.Log("Contacting Codex nodes...");
-            CheckCodexNodes(log, config);
-            log.Log("All OK.");
-            log.Log("");
 
             while (true)
             {
-                var run = new TestRun(config, log, testFactory);
+                var log = new FixtureLog(new LogConfig(config.LogPath, false), "StartupChecks");
+                var allTestsRun = new AllTestsRun(config, log, testFactory);
 
+                var result = ContinuousTestResult.Passed;
                 try
                 {
-                    run.Run();
+                    result = allTestsRun.RunAll();
                 }
                 catch (Exception ex)
                 {
                     log.Error($"Exception during test run: " + ex);
                 }
 
+                if (result == ContinuousTestResult.Failed)
+                {
+                    log.MarkAsFailed();
+                }
+
                 Thread.Sleep(config.SleepSecondsPerTest * 1000);
             }
         }
 
+        private void StartupChecks(Configuration config)
+        {
+            var log = new FixtureLog(new LogConfig(config.LogPath, false), "ContinuousTestsRun");
+            log.Log("Starting continuous test run...");
+            log.Log("Checking configuration...");
+            PreflightCheck(config);
+            log.Log("Contacting Codex nodes...");
+            CheckCodexNodes(log, config);
+            log.Log("All OK.");
+        }
+
         private void PreflightCheck(Configuration config)
         {
             var tests = testFactory.CreateTests();
@@ -106,4 +116,10 @@ namespace ContinuousTests
             return true;
         }
     }
+
+    public enum ContinuousTestResult
+    {
+        Passed,
+        Failed
+    }
 }

Logging/FixtureLog.cs

@@ -8,19 +8,19 @@ namespace Logging
         private readonly string fullName;
         private readonly LogConfig config;
 
-        public FixtureLog(LogConfig config)
+        public FixtureLog(LogConfig config, string name = "")
             : base(config.DebugEnabled)
         {
             start = DateTime.UtcNow;
             var folder = DetermineFolder(config);
-            var fixtureName = GetFixtureName();
+            var fixtureName = GetFixtureName(name);
             fullName = Path.Combine(folder, fixtureName);
             this.config = config;
         }
 
-        public TestLog CreateTestLog()
+        public TestLog CreateTestLog(string name = "")
         {
-            return new TestLog(fullName, config.DebugEnabled);
+            return new TestLog(fullName, config.DebugEnabled, name);
         }
 
         protected override LogFile CreateLogFile()
@@ -36,10 +36,12 @@ namespace Logging
                Pad(start.Day));
         }
 
-        private string GetFixtureName()
+        private string GetFixtureName(string name)
         {
             var test = TestContext.CurrentContext.Test;
             var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
+            if (!string.IsNullOrEmpty(name)) className = name;
+
             return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}";
         }
 

Logging/TestLog.cs

@@ -9,10 +9,10 @@ namespace Logging
         private readonly string methodName;
         private readonly string fullName;
 
-        public TestLog(string folder, bool debug)
+        public TestLog(string folder, bool debug, string name = "")
             : base(debug)
         {
-            methodName = GetMethodName();
+            methodName = GetMethodName(name);
             fullName = Path.Combine(folder, methodName);
 
             Log($"*** Begin: {methodName}");
@@ -39,13 +39,15 @@ namespace Logging
                 MarkAsFailed();
             }
         }
+
         protected override LogFile CreateLogFile()
         {
             return new LogFile(fullName, "log");
         }
 
-        private string GetMethodName()
+        private string GetMethodName(string name)
         {
+            if (!string.IsNullOrEmpty(name)) return name;
             var test = TestContext.CurrentContext.Test;
             var args = FormatArguments(test);
             return $"{test.MethodName}{args}";

Utils/RandomUtils.cs

@@ -0,0 +1,15 @@
+namespace Utils
+{
+    public static class RandomUtils
+    {
+        private static readonly Random random = new Random();
+
+        public static T PickOneRandom<T>(this List<T> remainingItems)
+        {
+            var i = random.Next(0, remainingItems.Count);
+            var result = remainingItems[i];
+            remainingItems.RemoveAt(i);
+            return result;
+        }
+    }
+}