From efc638a0f9a2156d92439144b7323022a2457887 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 28 Jun 2023 15:11:20 +0200 Subject: [PATCH] Debugging logging output --- ContinuousTests/ContinuousTest.cs | 2 +- ContinuousTests/ContinuousTestRunner.cs | 21 ++++++++++- ContinuousTests/K8sFactory.cs | 41 ++++++++++++++++++++++ ContinuousTests/NodeRunner.cs | 29 ++------------- ContinuousTests/Tests/MarketplaceTest.cs | 5 ++- ContinuousTests/Tests/PerformanceTests.cs | 2 +- ContinuousTests/Tests/TransientNodeTest.cs | 4 +-- ContinuousTests/Tests/TwoClientTest.cs | 2 +- DistTestCore/FileManager.cs | 1 - DistTestCore/Metrics/MetricsAccess.cs | 4 +-- DistTestCore/Metrics/MetricsDownloader.cs | 4 +-- DistTestCore/TestLifecycle.cs | 6 ++-- Logging/BaseLog.cs | 21 +++++++++-- Logging/FixtureLog.cs | 4 +-- Logging/NullLog.cs | 4 +-- Logging/TestLog.cs | 18 ++-------- 16 files changed, 104 insertions(+), 64 deletions(-) create mode 100644 ContinuousTests/K8sFactory.cs diff --git a/ContinuousTests/ContinuousTest.cs b/ContinuousTests/ContinuousTest.cs index 4ac0da1..9570dc1 100644 --- a/ContinuousTests/ContinuousTest.cs +++ b/ContinuousTests/ContinuousTest.cs @@ -77,7 +77,7 @@ namespace ContinuousTests return new ContentId(response); } - public TestFile DownloadContent(CodexNode node, ContentId contentId, string fileLabel = "") + public TestFile DownloadFile(CodexNode node, ContentId contentId, string fileLabel = "") { var logMessage = $"Downloading for contentId: '{contentId.Id}'..."; var file = FileManager.CreateEmptyTestFile(fileLabel); diff --git a/ContinuousTests/ContinuousTestRunner.cs b/ContinuousTests/ContinuousTestRunner.cs index d9cd276..85db06f 100644 --- a/ContinuousTests/ContinuousTestRunner.cs +++ b/ContinuousTests/ContinuousTestRunner.cs @@ -1,9 +1,11 @@ -using Logging; +using DistTestCore; +using Logging; namespace ContinuousTests { public class ContinuousTestRunner { + private readonly K8sFactory k8SFactory = new K8sFactory(); private readonly ConfigLoader configLoader = new ConfigLoader(); private readonly TestFactory testFactory = new TestFactory(); private readonly Configuration config; @@ -22,6 +24,9 @@ namespace ContinuousTests var overviewLog = new FixtureLog(new LogConfig(config.LogPath, false), "Overview"); overviewLog.Log("Continuous tests starting..."); var allTests = testFactory.CreateTests(); + + ClearAllCustomNamespaces(allTests, overviewLog); + var testLoop = allTests.Select(t => new TestLoop(config, overviewLog, t.GetType(), t.RunTestEvery)).ToArray(); foreach (var t in testLoop) @@ -34,5 +39,19 @@ namespace ContinuousTests overviewLog.Log("All test-loops launched."); while (true) Thread.Sleep((2 ^ 31) - 1); } + + private void ClearAllCustomNamespaces(ContinuousTest[] allTests, FixtureLog log) + { + foreach (var test in allTests) ClearAllCustomNamespaces(test, log); + } + + private void ClearAllCustomNamespaces(ContinuousTest test, FixtureLog log) + { + if (string.IsNullOrEmpty(test.CustomK8sNamespace)) return; + + log.Log($"Clearing namespace '{test.CustomK8sNamespace}'..."); + var (workflowCreator, _) = k8SFactory.CreateFacilities(config, test.CustomK8sNamespace, new DefaultTimeSet(), log); + workflowCreator.CreateWorkflow().DeleteTestResources(); + } } } diff --git a/ContinuousTests/K8sFactory.cs b/ContinuousTests/K8sFactory.cs new file mode 100644 index 0000000..040a4a3 --- /dev/null +++ b/ContinuousTests/K8sFactory.cs @@ -0,0 +1,41 @@ +using DistTestCore.Codex; +using DistTestCore; +using KubernetesWorkflow; +using Logging; + +namespace ContinuousTests +{ + public class K8sFactory + { + public (WorkflowCreator, TestLifecycle) CreateFacilities(Configuration config, string customNamespace, ITimeSet timeSet, BaseLog log) + { + var kubeConfig = GetKubeConfig(config.KubeConfigFile); + var lifecycleConfig = new DistTestCore.Configuration + ( + kubeConfigFile: kubeConfig, + logPath: "null", + logDebug: false, + dataFilesPath: config.LogPath, + codexLogLevel: CodexLogLevel.Debug, + runnerLocation: TestRunnerLocation.ExternalToCluster + ); + + var kubeFlowConfig = new KubernetesWorkflow.Configuration( + k8sNamespacePrefix: customNamespace, + kubeConfigFile: kubeConfig, + operationTimeout: timeSet.K8sOperationTimeout(), + retryDelay: timeSet.WaitForK8sServiceDelay()); + + var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty); + var lifecycle = new TestLifecycle(log, lifecycleConfig, timeSet, workflowCreator); + + return (workflowCreator, lifecycle); + } + + private static string? GetKubeConfig(string kubeConfigFile) + { + if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null; + return kubeConfigFile; + } + } +} diff --git a/ContinuousTests/NodeRunner.cs b/ContinuousTests/NodeRunner.cs index 92f17dc..1a36d62 100644 --- a/ContinuousTests/NodeRunner.cs +++ b/ContinuousTests/NodeRunner.cs @@ -10,6 +10,7 @@ namespace ContinuousTests { public class NodeRunner { + private readonly K8sFactory k8SFactory = new K8sFactory(); private readonly CodexNode[] nodes; private readonly Configuration config; private readonly ITimeSet timeSet; @@ -80,33 +81,7 @@ namespace ContinuousTests private (WorkflowCreator, TestLifecycle) CreateFacilities() { - var kubeConfig = GetKubeConfig(config.KubeConfigFile); - var lifecycleConfig = new DistTestCore.Configuration - ( - kubeConfigFile: kubeConfig, - logPath: "null", - logDebug: false, - dataFilesPath: config.LogPath, - codexLogLevel: CodexLogLevel.Debug, - runnerLocation: TestRunnerLocation.ExternalToCluster - ); - - var kubeFlowConfig = new KubernetesWorkflow.Configuration( - k8sNamespacePrefix: customNamespace, - kubeConfigFile: kubeConfig, - operationTimeout: timeSet.K8sOperationTimeout(), - retryDelay: timeSet.WaitForK8sServiceDelay()); - - var workflowCreator = new WorkflowCreator(log, kubeFlowConfig, testNamespacePostfix: string.Empty); - var lifecycle = new TestLifecycle(new NullLog(), lifecycleConfig, timeSet, workflowCreator); - - return (workflowCreator, lifecycle); - } - - private static string? GetKubeConfig(string kubeConfigFile) - { - if (string.IsNullOrEmpty(kubeConfigFile) || kubeConfigFile.ToLowerInvariant() == "null") return null; - return kubeConfigFile; + return k8SFactory.CreateFacilities(config, customNamespace, timeSet, log); } } } diff --git a/ContinuousTests/Tests/MarketplaceTest.cs b/ContinuousTests/Tests/MarketplaceTest.cs index 7dd1a26..9582b66 100644 --- a/ContinuousTests/Tests/MarketplaceTest.cs +++ b/ContinuousTests/Tests/MarketplaceTest.cs @@ -2,6 +2,7 @@ using DistTestCore.Codex; using Newtonsoft.Json; using NUnit.Framework; +using Utils; namespace ContinuousTests.Tests { @@ -54,7 +55,7 @@ namespace ContinuousTests.Tests { NodeRunner.RunNode((codexAccess, marketplaceAccess) => { - var result = DownloadContent(codexAccess.Node, cid!); + var result = DownloadFile(codexAccess.Node, cid!); file.AssertIsEqual(result); }); @@ -67,6 +68,7 @@ namespace ContinuousTests.Tests var filesizeInMb = fileSize.SizeInBytes / 1024; var maxWaitTime = TimeSpan.FromSeconds(filesizeInMb * 10.0); + Log.Log(nameof(WaitForContractToStart) + " for " + Time.FormatDuration(maxWaitTime)); while (lastState != "started") { var purchaseStatus = codexAccess.Node.GetPurchaseStatus(purchaseId); @@ -87,6 +89,7 @@ namespace ContinuousTests.Tests Assert.Fail($"Contract was not picked up within {maxWaitTime.TotalSeconds} seconds timeout: " + JsonConvert.SerializeObject(purchaseStatus)); } } + Log.Log("Contract started."); } } } diff --git a/ContinuousTests/Tests/PerformanceTests.cs b/ContinuousTests/Tests/PerformanceTests.cs index 53af7be..032dcfa 100644 --- a/ContinuousTests/Tests/PerformanceTests.cs +++ b/ContinuousTests/Tests/PerformanceTests.cs @@ -66,7 +66,7 @@ namespace ContinuousTests.Tests TestFile? result = null; var time = Measure(() => { - result = DownloadContent(downloadNode, cid!); + result = DownloadFile(downloadNode, cid!); }); file.AssertIsEqual(result); diff --git a/ContinuousTests/Tests/TransientNodeTest.cs b/ContinuousTests/Tests/TransientNodeTest.cs index 24971e2..5c49e1b 100644 --- a/ContinuousTests/Tests/TransientNodeTest.cs +++ b/ContinuousTests/Tests/TransientNodeTest.cs @@ -29,7 +29,7 @@ namespace ContinuousTests.Tests cid = UploadFile(codexAccess.Node, file)!; Assert.That(cid, Is.Not.Null); - var resultFile = DownloadContent(IntermediateNode, cid); + var resultFile = DownloadFile(IntermediateNode, cid); file.AssertIsEqual(resultFile); }); } @@ -39,7 +39,7 @@ namespace ContinuousTests.Tests { NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess) => { - var resultFile = DownloadContent(codexAccess.Node, cid); + var resultFile = DownloadFile(codexAccess.Node, cid); file.AssertIsEqual(resultFile); }); } diff --git a/ContinuousTests/Tests/TwoClientTest.cs b/ContinuousTests/Tests/TwoClientTest.cs index e6cbc1a..9d23d77 100644 --- a/ContinuousTests/Tests/TwoClientTest.cs +++ b/ContinuousTests/Tests/TwoClientTest.cs @@ -24,7 +24,7 @@ namespace ContinuousTests.Tests [TestMoment(t: MinuteFive)] public void DownloadTestFile() { - var dl = DownloadContent(Nodes[1], cid!); + var dl = DownloadFile(Nodes[1], cid!); file.AssertIsEqual(dl); } diff --git a/DistTestCore/FileManager.cs b/DistTestCore/FileManager.cs index 8d8c55f..ab86473 100644 --- a/DistTestCore/FileManager.cs +++ b/DistTestCore/FileManager.cs @@ -1,6 +1,5 @@ using Logging; using NUnit.Framework; -using System.Runtime.InteropServices; using Utils; namespace DistTestCore diff --git a/DistTestCore/Metrics/MetricsAccess.cs b/DistTestCore/Metrics/MetricsAccess.cs index 5ea0940..23b6522 100644 --- a/DistTestCore/Metrics/MetricsAccess.cs +++ b/DistTestCore/Metrics/MetricsAccess.cs @@ -14,12 +14,12 @@ namespace DistTestCore.Metrics public class MetricsAccess : IMetricsAccess { - private readonly TestLog log; + private readonly BaseLog log; private readonly ITimeSet timeSet; private readonly MetricsQuery query; private readonly RunningContainer node; - public MetricsAccess(TestLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node) + public MetricsAccess(BaseLog log, ITimeSet timeSet, MetricsQuery query, RunningContainer node) { this.log = log; this.timeSet = timeSet; diff --git a/DistTestCore/Metrics/MetricsDownloader.cs b/DistTestCore/Metrics/MetricsDownloader.cs index 4a458dd..d0d11cd 100644 --- a/DistTestCore/Metrics/MetricsDownloader.cs +++ b/DistTestCore/Metrics/MetricsDownloader.cs @@ -5,9 +5,9 @@ namespace DistTestCore.Metrics { public class MetricsDownloader { - private readonly TestLog log; + private readonly BaseLog log; - public MetricsDownloader(TestLog log) + public MetricsDownloader(BaseLog log) { this.log = log; } diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index 6ea8441..5249960 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -9,12 +9,12 @@ namespace DistTestCore { private DateTime testStart = DateTime.MinValue; - public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet) + public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet) : this(log, configuration, timeSet, new WorkflowCreator(log, configuration.GetK8sConfiguration(timeSet))) { } - public TestLifecycle(TestLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator) + public TestLifecycle(BaseLog log, Configuration configuration, ITimeSet timeSet, WorkflowCreator workflowCreator) { Log = log; Configuration = configuration; @@ -27,7 +27,7 @@ namespace DistTestCore testStart = DateTime.UtcNow; } - public TestLog Log { get; } + public BaseLog Log { get; } public Configuration Configuration { get; } public ITimeSet TimeSet { get; } public FileManager FileManager { get; } diff --git a/Logging/BaseLog.cs b/Logging/BaseLog.cs index c312256..14656e3 100644 --- a/Logging/BaseLog.cs +++ b/Logging/BaseLog.cs @@ -4,6 +4,7 @@ namespace Logging { public abstract class BaseLog { + private readonly NumberSource subfileNumberSource = new NumberSource(0); private readonly bool debug; private readonly List replacements = new List(); private bool hasFailed; @@ -14,17 +15,21 @@ namespace Logging this.debug = debug; } - protected abstract LogFile CreateLogFile(); + protected abstract string GetFullName(); - protected LogFile LogFile + public LogFile LogFile { get { - if (logFile == null) logFile = CreateLogFile(); + if (logFile == null) logFile = new LogFile(GetFullName(), "log"); return logFile; } } + public virtual void EndTest() + { + } + public virtual void Log(string message) { LogFile.Write(ApplyReplacements(message)); @@ -63,6 +68,11 @@ namespace Logging File.Delete(LogFile.FullFilename); } + public LogFile CreateSubfile(string ext = "log") + { + return new LogFile($"{GetFullName()}_{GetSubfileNumber()}", ext); + } + private string ApplyReplacements(string str) { foreach (var replacement in replacements) @@ -71,6 +81,11 @@ namespace Logging } return str; } + + private string GetSubfileNumber() + { + return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0'); + } } public class BaseLogStringReplacement diff --git a/Logging/FixtureLog.cs b/Logging/FixtureLog.cs index cc5ea87..e386f01 100644 --- a/Logging/FixtureLog.cs +++ b/Logging/FixtureLog.cs @@ -28,9 +28,9 @@ namespace Logging Directory.Delete(fullName, true); } - protected override LogFile CreateLogFile() + protected override string GetFullName() { - return new LogFile(fullName, "log"); + return fullName; } private string DetermineFolder(LogConfig config) diff --git a/Logging/NullLog.cs b/Logging/NullLog.cs index 5323508..06113b1 100644 --- a/Logging/NullLog.cs +++ b/Logging/NullLog.cs @@ -6,9 +6,9 @@ { } - protected override LogFile CreateLogFile() + protected override string GetFullName() { - return null!; + return "NULL"; } public override void Log(string message) diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs index 6ac0c99..ed2cdfd 100644 --- a/Logging/TestLog.cs +++ b/Logging/TestLog.cs @@ -1,11 +1,9 @@ using NUnit.Framework; -using Utils; namespace Logging { public class TestLog : BaseLog { - private readonly NumberSource subfileNumberSource = new NumberSource(0); private readonly string methodName; private readonly string fullName; @@ -18,12 +16,7 @@ namespace Logging Log($"*** Begin: {methodName}"); } - public LogFile CreateSubfile(string ext = "log") - { - return new LogFile($"{fullName}_{GetSubfileNumber()}", ext); - } - - public void EndTest() + public override void EndTest() { var result = TestContext.CurrentContext.Result; @@ -40,9 +33,9 @@ namespace Logging } } - protected override LogFile CreateLogFile() + protected override string GetFullName() { - return new LogFile(fullName, "log"); + return fullName; } private string GetMethodName(string name) @@ -53,11 +46,6 @@ namespace Logging return $"{test.MethodName}{args}"; } - private string GetSubfileNumber() - { - return subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0'); - } - private static string FormatArguments(TestContext.TestAdapter test) { if (test.Arguments == null || !test.Arguments.Any()) return "";