Debugging logging output

ContinuousTests/ContinuousTest.cs

@@ -77,7 +77,7 @@ namespace ContinuousTests
             return new ContentId(response);
         }
 
-        public TestFile DownloadContent(CodexNode node, ContentId contentId, string fileLabel = "")
+        public TestFile DownloadFile(CodexNode node, ContentId contentId, string fileLabel = "")
         {
             var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
             var file = FileManager.CreateEmptyTestFile(fileLabel);

ContinuousTests/ContinuousTestRunner.cs

@@ -1,9 +1,11 @@
-using Logging;
+using DistTestCore;
+using Logging;
 
 namespace ContinuousTests
 {
     public class ContinuousTestRunner
     {
+        private readonly K8sFactory k8SFactory = new K8sFactory();
         private readonly ConfigLoader configLoader = new ConfigLoader();
         private readonly TestFactory testFactory = new TestFactory();
         private readonly Configuration config;
@@ -22,6 +24,9 @@ namespace ContinuousTests
             var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview");
             overviewLog.Log("Continuous tests starting...");
             var allTests = testFactory.CreateTests();
+
+            ClearAllCustomNamespaces(allTests, overviewLog);
+
             var testLoop = allTests.Select(t => new TestLoop(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray();
 
             foreach (var t in testLoop)
@@ -34,5 +39,19 @@ namespace ContinuousTests
             overviewLog.Log("All test-loops launched.");
             while (true) Thread.Sleep((2 ^ 31) - 1);
         }
+
+        private void ClearAllCustomNamespaces(ContinuousTest[] allTests, FixtureLog log)
+        {
+            foreach (var test in allTests) ClearAllCustomNamespaces(test, log);
+        }
+
+        private void ClearAllCustomNamespaces(ContinuousTest test, FixtureLog log)
+        {
+            if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return;
+
+            log.Log($"Clearing namespace '{test.CustomK8sNamespace}'...");
+            var (workflowCreator, _) = k8SFactory.CreateFacilities(config, test.CustomK8sNamespace, new DefaultTimeSet(), log);
+            workflowCreator.CreateWorkflow().DeleteTestResources();
+        }
     }
 }

ContinuousTests/K8sFactory.cs

@@ -0,0 +1,41 @@
+using DistTestCore.Codex;
+using DistTestCore;
+using KubernetesWorkflow;
+using Logging;
+
+namespace ContinuousTests
+{
+    public class K8sFactory
+    {
+        public (WorkflowCreator, TestLifecycle) CreateFacilities(Configuration config, string customNamespace, ITimeSet timeSet, BaseLog log)
+        {
+            var kubeConfig = GetKubeConfig(config.KubeConfigFile);
+            var lifecycleConfig = new DistTestCore.Configuration
+            (
+                kubeConfigFile: kubeConfig,
+                logPath: "null",
+                logDebug: false,
+                dataFilesPath: config.LogPath,
+                codexLogLevel: CodexLogLevel.Debug,
+                runnerLocation: TestRunnerLocation.ExternalToCluster
+            );
+
+            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
+                k8sNamespacePrefix: customNamespace,
+            kubeConfigFile: kubeConfig,
+                operationTimeout: timeSet.K8sOperationTimeout(),
+            retryDelay: timeSet.WaitForK8sServiceDelay());
+
+            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
+            var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator);
+
+            return (workflowCreator, lifecycle);
+        }
+
+        private static string? GetKubeConfig(string kubeConfigFile)
+        {
+            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
+            return kubeConfigFile;
+        }
+    }
+}

ContinuousTests/NodeRunner.cs

@@ -10,6 +10,7 @@ namespace ContinuousTests
 {
     public class NodeRunner
     {
+        private readonly K8sFactory k8SFactory = new K8sFactory();
         private readonly CodexNode[] nodes;
         private readonly Configuration config;
         private readonly ITimeSet timeSet;
@@ -80,33 +81,7 @@ namespace ContinuousTests
 
         private (WorkflowCreator, TestLifecycle) CreateFacilities()
         {
-            var kubeConfig = GetKubeConfig(config.KubeConfigFile);
-            var lifecycleConfig = new DistTestCore.Configuration
-            (
-                kubeConfigFile: kubeConfig,
-                logPath: "null",
-                logDebug: false,
-                dataFilesPath: config.LogPath,
-                codexLogLevel: CodexLogLevel.Debug,
-                runnerLocation: TestRunnerLocation.ExternalToCluster
-            );
-
-            var kubeFlowConfig = new KubernetesWorkflow.Configuration(
-                k8sNamespacePrefix: customNamespace,
-                kubeConfigFile: kubeConfig,
-                operationTimeout: timeSet.K8sOperationTimeout(),
-            retryDelay: timeSet.WaitForK8sServiceDelay());
-
-            var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty);
-            var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, timeSet, workflowCreator);
-
-            return (workflowCreator, lifecycle);
-        }
-
-        private static string? GetKubeConfig(string kubeConfigFile)
-        {
-            if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null;
-            return kubeConfigFile;
+            return k8SFactory.CreateFacilities(config, customNamespace, timeSet, log);
         }
     }
 }

ContinuousTests/Tests/MarketplaceTest.cs

@@ -2,6 +2,7 @@
 using DistTestCore.Codex;
 using Newtonsoft.Json;
 using NUnit.Framework;
+using Utils;
 
 namespace ContinuousTests.Tests
 {
@@ -54,7 +55,7 @@ namespace ContinuousTests.Tests
         {
             NodeRunner.RunNode((codexAccess, marketplaceAccess) =>
             {
-                var result = DownloadContent(codexAccess.Node, cid!);
+                var result = DownloadFile(codexAccess.Node, cid!);
 
                 file.AssertIsEqual(result);
             });
@@ -67,6 +68,7 @@ namespace ContinuousTests.Tests
             var filesizeInMb = fileSize.SizeInBytes / 1024;
             var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0);
 
+            Log.Log(nameof(WaitForContractToStart) + " for " + Time.FormatDuration(maxWaitTime));
             while (lastState != "started")
             {
                 var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId);
@@ -87,6 +89,7 @@ namespace ContinuousTests.Tests
                     Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: " + JsonConvert.SerializeObject(purchaseStatus));
                 }
             }
+            Log.Log("Contract started.");
         }
     }
 }

ContinuousTests/Tests/PerformanceTests.cs

@@ -66,7 +66,7 @@ namespace ContinuousTests.Tests
             TestFile? result = null;
             var time = Measure(() =>
             {
-                result = DownloadContent(downloadNode, cid!);
+                result = DownloadFile(downloadNode, cid!);
             });
 
             file.AssertIsEqual(result);

ContinuousTests/Tests/TransientNodeTest.cs

@@ -29,7 +29,7 @@ namespace ContinuousTests.Tests
                 cid = UploadFile(codexAccess.Node, file)!;
                 Assert.That(cid, Is.Not.Null);
 
-                var resultFile = DownloadContent(IntermediateNode, cid);
+                var resultFile = DownloadFile(IntermediateNode, cid);
                 file.AssertIsEqual(resultFile);
             });
         }
@@ -39,7 +39,7 @@ namespace ContinuousTests.Tests
         {
             NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess) =>
             {
-                var resultFile = DownloadContent(codexAccess.Node, cid);
+                var resultFile = DownloadFile(codexAccess.Node, cid);
                 file.AssertIsEqual(resultFile);
             });
         }

ContinuousTests/Tests/TwoClientTest.cs

@@ -24,7 +24,7 @@ namespace ContinuousTests.Tests
         [TestMoment(t: MinuteFive)]
         public void DownloadTestFile()
         {
-            var dl = DownloadContent(Nodes[1], cid!);
+            var dl = DownloadFile(Nodes[1], cid!);
 
             file.AssertIsEqual(dl);
         }

DistTestCore/FileManager.cs

@@ -1,6 +1,5 @@
 using Logging;
 using NUnit.Framework;
-using System.Runtime.InteropServices;
 using Utils;
 
 namespace DistTestCore

DistTestCore/Metrics/MetricsAccess.cs

@@ -14,12 +14,12 @@ namespace DistTestCore.Metrics
 
     public class MetricsAccess : IMetricsAccess
     {
-        private readonly TestLog log;
+        private readonly BaseLog log;
         private readonly ITimeSet timeSet;
         private readonly MetricsQuery query;
         private readonly RunningContainer node;
 
-        public MetricsAccess(TestLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
+        public MetricsAccess(BaseLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node)
         {
             this.log = log;
             this.timeSet = timeSet;

DistTestCore/Metrics/MetricsDownloader.cs

@@ -5,9 +5,9 @@ namespace DistTestCore.Metrics
 {
     public class MetricsDownloader
     {
-        private readonly TestLog log;
+        private readonly BaseLog log;
 
-        public MetricsDownloader(TestLog log)
+        public MetricsDownloader(BaseLog log)
         {
             this.log = log;
         }

DistTestCore/TestLifecycle.cs

@@ -9,12 +9,12 @@ namespace DistTestCore
     {
         private DateTime testStart = DateTime.MinValue;
 
-        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet)
+        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet)
             : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet)))
         {
         }
 
-        public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
+        public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator)
         {
             Log = log;
             Configuration = configuration;
@@ -27,7 +27,7 @@ namespace DistTestCore
             testStart = DateTime.UtcNow;
         }
 
-        public TestLog Log { get; }
+        public BaseLog Log { get; }
         public Configuration Configuration { get; }
         public ITimeSet TimeSet { get; }
         public FileManager FileManager { get; }

Logging/BaseLog.cs

@@ -4,6 +4,7 @@ namespace Logging
 {
     public abstract class BaseLog
     {
+        private readonly NumberSource subfileNumberSource = new NumberSource(0);
         private readonly bool debug;
         private readonly List<BaseLogStringReplacement> replacements = new List<BaseLogStringReplacement>();
         private bool hasFailed;
@@ -14,17 +15,21 @@ namespace Logging
             this.debug = debug;
         }
 
-        protected abstract LogFile CreateLogFile();
+        protected abstract string GetFullName();
 
-        protected LogFile LogFile 
+        public LogFile LogFile 
         {
             get
             {
-                if (logFile == null) logFile = CreateLogFile();
+                if (logFile == null) logFile = new LogFile(GetFullName(), "log");
                 return logFile;
             }
         }
 
+        public virtual void EndTest()
+        {
+        }
+
         public virtual void Log(string message)
         {
             LogFile.Write(ApplyReplacements(message));
@@ -63,6 +68,11 @@ namespace Logging
             File.Delete(LogFile.FullFilename);
         }
 
+        public LogFile CreateSubfile(string ext = "log")
+        {
+            return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext);
+        }
+
         private string ApplyReplacements(string str)
         {
             foreach (var replacement in replacements)
@@ -71,6 +81,11 @@ namespace Logging
             }
             return str;
         }
+
+        private string GetSubfileNumber()
+        {
+            return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0');
+        }
     }
 
     public class BaseLogStringReplacement

Logging/FixtureLog.cs

@@ -28,9 +28,9 @@ namespace Logging
             Directory.Delete(fullName, true);
         }
 
-        protected override LogFile CreateLogFile()
+        protected override string GetFullName()
         {
-            return new LogFile(fullName, "log");
+            return fullName;
         }
 
         private string DetermineFolder(LogConfig config)

Logging/NullLog.cs

@@ -6,9 +6,9 @@
         {
         }
 
-        protected override LogFile CreateLogFile()
+        protected override string GetFullName()
         {
-            return null!;
+            return "NULL";
         }
 
         public override void Log(string message)

Logging/TestLog.cs

@@ -1,11 +1,9 @@
 using NUnit.Framework;
-using Utils;
 
 namespace Logging
 {
     public class TestLog : BaseLog
     {
-        private readonly NumberSource subfileNumberSource = new NumberSource(0);
         private readonly string methodName;
         private readonly string fullName;
 
@@ -18,12 +16,7 @@ namespace Logging
             Log($"*** Begin: {methodName}");
         }
 
-        public LogFile CreateSubfile(string ext = "log")
-        {
-            return new LogFile($"{fullName}_{GetSubfileNumber()}", ext);
-        }
-
-        public void EndTest()
+        public override void EndTest()
         {
             var result = TestContext.CurrentContext.Result;
 
@@ -40,9 +33,9 @@ namespace Logging
             }
         }
 
-        protected override LogFile CreateLogFile()
+        protected override string GetFullName()
         {
-            return new LogFile(fullName, "log");
+            return fullName;
         }
 
         private string GetMethodName(string name)
@@ -53,11 +46,6 @@ namespace Logging
             return $"{test.MethodName}{args}";
         }
 
-        private string GetSubfileNumber()
-        {
-            return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0');
-        }
-
         private static string FormatArguments(TestContext.TestAdapter test)
         {
             if (test.Arguments == null || !test.Arguments.Any()) return "";