From 02ef655a40dcd5014f89fd6a1876346697c7b2a7 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 22 Apr 2025 12:25:37 +0200 Subject: [PATCH] Enables console output for release tests. --- Framework/Logging/BaseLog.cs | 10 ++- Framework/Logging/ConsoleLog.cs | 2 +- Framework/Logging/FileLog.cs | 2 +- Framework/Logging/LogPrefixer.cs | 11 ++- Framework/Logging/LogSplitter.cs | 10 +++ Framework/Logging/NullLog.cs | 6 +- .../ContinuousTestRunner.cs | 2 +- Tests/CodexContinuousTests/SingleTestRun.cs | 8 +- Tests/CodexContinuousTests/StartupChecker.cs | 4 +- Tests/DistTestCore/DistTest.cs | 24 +++--- Tests/DistTestCore/Logs/BaseTestLog.cs | 74 ++++++++++++++++--- Tests/DistTestCore/Logs/FixtureLog.cs | 22 +++--- Tests/DistTestCore/Logs/TestLog.cs | 22 +++--- Tests/ExperimentalTests/CodexDistTest.cs | 6 +- 14 files changed, 138 insertions(+), 65 deletions(-) diff --git a/Framework/Logging/BaseLog.cs b/Framework/Logging/BaseLog.cs index 2c79fd39..56ff66ec 100644 --- a/Framework/Logging/BaseLog.cs +++ b/Framework/Logging/BaseLog.cs @@ -7,8 +7,10 @@ namespace Logging void Log(string message); void Debug(string message = "", int skipFrames = 0); void Error(string message); + void Raw(string message); void AddStringReplace(string from, string to); LogFile CreateSubfile(string addName, string ext = "log"); + string GetFullName(); } public abstract class BaseLog : ILog @@ -28,7 +30,8 @@ namespace Logging } protected bool IsDebug { get; private set; } - protected abstract string GetFullName(); + + public abstract string GetFullName(); public LogFile LogFile { @@ -60,6 +63,11 @@ namespace Logging Log(msg); } + public void Raw(string message) + { + LogFile.WriteRaw(message); + } + public virtual void AddStringReplace(string from, string to) { if (string.IsNullOrWhiteSpace(from)) return; diff --git a/Framework/Logging/ConsoleLog.cs b/Framework/Logging/ConsoleLog.cs index ab67a400..2604e64f 100644 --- a/Framework/Logging/ConsoleLog.cs +++ b/Framework/Logging/ConsoleLog.cs @@ -2,7 +2,7 @@ { public class ConsoleLog : BaseLog { - protected override string GetFullName() + public override string GetFullName() { return "CONSOLE"; } diff --git a/Framework/Logging/FileLog.cs b/Framework/Logging/FileLog.cs index cc5e9952..86bc1985 100644 --- a/Framework/Logging/FileLog.cs +++ b/Framework/Logging/FileLog.cs @@ -9,7 +9,7 @@ public string FullFilename { get; } - protected override string GetFullName() + public override string GetFullName() { return FullFilename; } diff --git a/Framework/Logging/LogPrefixer.cs b/Framework/Logging/LogPrefixer.cs index 6fe6033b..f0f303d6 100644 --- a/Framework/Logging/LogPrefixer.cs +++ b/Framework/Logging/LogPrefixer.cs @@ -17,7 +17,6 @@ public string Prefix { get; set; } = string.Empty; - public LogFile CreateSubfile(string addName, string ext = "log") { return backingLog.CreateSubfile(addName, ext); @@ -42,5 +41,15 @@ { backingLog.AddStringReplace(from, to); } + + public void Raw(string message) + { + backingLog.Raw(message); + } + + public string GetFullName() + { + return backingLog.GetFullName(); + } } } diff --git a/Framework/Logging/LogSplitter.cs b/Framework/Logging/LogSplitter.cs index 563f5812..8de462c9 100644 --- a/Framework/Logging/LogSplitter.cs +++ b/Framework/Logging/LogSplitter.cs @@ -29,11 +29,21 @@ OnAll(l => l.Error(message)); } + public string GetFullName() + { + return targetLogs.First().GetFullName(); + } + public void Log(string message) { OnAll(l => l.Log(message)); } + public void Raw(string message) + { + OnAll(l => l.Raw(message)); + } + private void OnAll(Action action) { foreach (var t in targetLogs) action(t); diff --git a/Framework/Logging/NullLog.cs b/Framework/Logging/NullLog.cs index a9cf8d2d..f8094f32 100644 --- a/Framework/Logging/NullLog.cs +++ b/Framework/Logging/NullLog.cs @@ -2,11 +2,9 @@ { public class NullLog : BaseLog { - public string FullFilename { get; set; } = "NULL"; - - protected override string GetFullName() + public override string GetFullName() { - return FullFilename; + return "NULL"; } public override void Log(string message) diff --git a/Tests/CodexContinuousTests/ContinuousTestRunner.cs b/Tests/CodexContinuousTests/ContinuousTestRunner.cs index fde65aa3..edec4e43 100644 --- a/Tests/CodexContinuousTests/ContinuousTestRunner.cs +++ b/Tests/CodexContinuousTests/ContinuousTestRunner.cs @@ -27,7 +27,7 @@ namespace ContinuousTests var startTime = DateTime.UtcNow; var overviewLog = new LogSplitter( - new FixtureLog(logConfig, startTime, config.CodexDeployment.Id, "Overview"), + FixtureLog.Create(logConfig, startTime, config.CodexDeployment.Id, "Overview"), new ConsoleLog() ); var statusLog = new StatusLog(logConfig, startTime, "continuous-tests", config.CodexDeployment.Id, diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/CodexContinuousTests/SingleTestRun.cs index 0606c8e4..93870406 100644 --- a/Tests/CodexContinuousTests/SingleTestRun.cs +++ b/Tests/CodexContinuousTests/SingleTestRun.cs @@ -37,7 +37,7 @@ namespace ContinuousTests this.handle = handle; this.cancelToken = cancelToken; testName = handle.Test.GetType().Name; - fixtureLog = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, deployId, testName); + fixtureLog = FixtureLog.Create(new LogConfig(config.LogPath), DateTime.UtcNow, deployId, testName); entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, fixtureLog); ApplyLogReplacements(fixtureLog, startupChecker); @@ -81,17 +81,11 @@ namespace ContinuousTests OverviewLog($" > Test passed. ({Time.FormatDuration(duration)})"); UpdateStatusLogPassed(testStart, duration); - if (!config.KeepPassedTestLogs) - { - fixtureLog.Delete(); - } - resultHandler(true); } catch (Exception ex) { fixtureLog.Error("Test run failed with exception: " + ex); - fixtureLog.MarkAsFailed(); UpdateStatusLogFailed(testStart, duration, ex.ToString()); DownloadContainerLogs(testStart); diff --git a/Tests/CodexContinuousTests/StartupChecker.cs b/Tests/CodexContinuousTests/StartupChecker.cs index 0914b3d1..2febf14e 100644 --- a/Tests/CodexContinuousTests/StartupChecker.cs +++ b/Tests/CodexContinuousTests/StartupChecker.cs @@ -22,7 +22,7 @@ namespace ContinuousTests public void Check() { - var log = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, config.CodexDeployment.Id, + var log = FixtureLog.Create(new LogConfig(config.LogPath), DateTime.UtcNow, config.CodexDeployment.Id, "StartupChecks"); log.Log("Starting continuous test run..."); IncludeDeploymentConfiguration(log); @@ -90,7 +90,7 @@ namespace ContinuousTests } } - private void CheckCodexNodes(BaseLog log, Configuration config) + private void CheckCodexNodes(ILog log, Configuration config) { throw new NotImplementedException(); diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index 691f19ed..534eb74a 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -33,7 +33,7 @@ namespace DistTestCore var logConfig = configuration.GetLogConfig(); var startTime = DateTime.UtcNow; - fixtureLog = new FixtureLog(logConfig, startTime, deployId); + fixtureLog = FixtureLog.Create(logConfig, startTime, deployId); statusLog = new StatusLog(logConfig, startTime, "dist-tests", deployId); globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultK8sTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder()); @@ -44,7 +44,7 @@ namespace DistTestCore [OneTimeSetUp] public void GlobalSetup() { - fixtureLog.Log($"Distributed Tests are starting..."); + fixtureLog.Log($"Starting..."); globalEntryPoint.Announce(); // Previous test run may have been interrupted. @@ -87,7 +87,15 @@ namespace DistTestCore } else { - CreateNewTestLifecycle(); + try + { + CreateNewTestLifecycle(); + } + catch (Exception ex) + { + fixtureLog.Error("Setup failed: " + ex); + GlobalTestFailure.HasFailed = true; + } } } @@ -236,11 +244,6 @@ namespace DistTestCore Log(result.Message); Log($"{result.StackTrace}"); } - - if (result.Outcome.Status == TestStatus.Failed) - { - log.MarkAsFailed(); - } } private IWebCallTimeSet GetWebCallTimeSet() @@ -296,11 +299,6 @@ namespace DistTestCore private void IncludeLogsOnTestFailure(TestLifecycle lifecycle) { var testStatus = TestContext.CurrentContext.Result.Outcome.Status; - if (testStatus == TestStatus.Failed) - { - fixtureLog.MarkAsFailed(); - } - if (ShouldDownloadAllLogs(testStatus)) { lifecycle.Log.Log("Downloading all container logs..."); diff --git a/Tests/DistTestCore/Logs/BaseTestLog.cs b/Tests/DistTestCore/Logs/BaseTestLog.cs index 22485545..e670e0c1 100644 --- a/Tests/DistTestCore/Logs/BaseTestLog.cs +++ b/Tests/DistTestCore/Logs/BaseTestLog.cs @@ -2,27 +2,83 @@ namespace DistTestCore.Logs { - public abstract class BaseTestLog : BaseLog + public abstract class BaseTestLog : ILog { - private bool hasFailed; - private readonly string deployId; + private readonly ILog backingLog; - protected BaseTestLog(string deployId) + protected BaseTestLog(ILog backingLog, string deployId) { - this.deployId = deployId; + this.backingLog = backingLog; + + DeployId = deployId; + } + + public string DeployId { get; } + + public void AddStringReplace(string from, string to) + { + backingLog.AddStringReplace(from, to); + } + + public LogFile CreateSubfile(string addName, string ext = "log") + { + return backingLog.CreateSubfile(addName, ext); + } + + public void Debug(string message = "", int skipFrames = 0) + { + backingLog.Debug(message, skipFrames); + } + + public void Error(string message) + { + backingLog.Error(message); + } + + public string GetFullName() + { + return backingLog.GetFullName(); + } + + public void Log(string message) + { + backingLog.Log(message); + } + + public void Raw(string message) + { + backingLog.Raw(message); } public void WriteLogTag() { var category = NameUtils.GetCategoryName(); var name = NameUtils.GetTestMethodName(); - LogFile.WriteRaw($"{deployId} {category} {name}"); + backingLog.Raw($"{DeployId} {category} {name}"); } - public void MarkAsFailed() + protected static ILog CreateMainLog(string fullName, string name) { - if (hasFailed) return; - hasFailed = true; + ILog log = new FileLog(fullName); + log = ApplyConsoleOutput(log); + return log; + } + + private static ILog ApplyConsoleOutput(ILog log) + { + // If we're running as a release test, we'll split the log output + // to the console as well. + + var testType = Environment.GetEnvironmentVariable("TEST_TYPE"); + if (string.IsNullOrEmpty(testType) || testType.ToLowerInvariant() != "release-tests") + { + return log; + } + + return new LogSplitter( + log, + new ConsoleLog() + ); } } } diff --git a/Tests/DistTestCore/Logs/FixtureLog.cs b/Tests/DistTestCore/Logs/FixtureLog.cs index 6559f81d..9d3c77d3 100644 --- a/Tests/DistTestCore/Logs/FixtureLog.cs +++ b/Tests/DistTestCore/Logs/FixtureLog.cs @@ -4,28 +4,26 @@ namespace DistTestCore.Logs { public class FixtureLog : BaseTestLog { - private readonly string fullName; + private readonly ILog backingLog; private readonly string deployId; - public FixtureLog(LogConfig config, DateTime start, string deployId, string name = "") : base(deployId) + public FixtureLog(ILog backingLog, string deployId) + : base(backingLog, deployId) { + this.backingLog = backingLog; this.deployId = deployId; - fullName = NameUtils.GetFixtureFullName(config, start, name); } public TestLog CreateTestLog(string name = "") { - return new TestLog(fullName, deployId, name); + return TestLog.Create(this, name); } - public void DeleteFolder() + public static FixtureLog Create(LogConfig config, DateTime start, string deployId, string name = "") { - Directory.Delete(fullName, true); - } - - protected override string GetFullName() - { - return fullName; + var fullName = NameUtils.GetFixtureFullName(config, start, name); + var log = CreateMainLog(fullName, name); + return new FixtureLog(log, deployId); } } -} \ No newline at end of file +} diff --git a/Tests/DistTestCore/Logs/TestLog.cs b/Tests/DistTestCore/Logs/TestLog.cs index 1f598b65..0dca1464 100644 --- a/Tests/DistTestCore/Logs/TestLog.cs +++ b/Tests/DistTestCore/Logs/TestLog.cs @@ -1,20 +1,22 @@ -namespace DistTestCore.Logs +using Logging; +using System.Xml.Linq; + +namespace DistTestCore.Logs { public class TestLog : BaseTestLog { - private readonly string fullName; - - public TestLog(string folder, string deployId, string name = "") : base(deployId) + public TestLog(ILog backingLog, string methodName, string deployId, string name = "") + : base(backingLog, deployId) { - var methodName = NameUtils.GetTestMethodName(name); - fullName = Path.Combine(folder, methodName); - - Log($"*** Begin: {methodName}"); + backingLog.Log($"*** Begin: {methodName}"); } - protected override string GetFullName() + public static TestLog Create(FixtureLog parentLog, string name = "") { - return fullName; + var methodName = NameUtils.GetTestMethodName(name); + var fullName = Path.Combine(parentLog.GetFullName(), methodName); + var backingLog = CreateMainLog(fullName, name); + return new TestLog(backingLog, methodName, parentLog.DeployId); } } } diff --git a/Tests/ExperimentalTests/CodexDistTest.cs b/Tests/ExperimentalTests/CodexDistTest.cs index 058d5bfd..d27216d3 100644 --- a/Tests/ExperimentalTests/CodexDistTest.cs +++ b/Tests/ExperimentalTests/CodexDistTest.cs @@ -301,7 +301,7 @@ namespace CodexTests Stopwatch.Measure(lifecycle.Log, $"Transcript.Finalize: {outputFilepath}", () => { - writer.IncludeFile(lifecycle.Log.LogFile.Filename); + writer.IncludeFile(lifecycle.Log.GetFullName()); writer.Finalize(outputFilepath); }); } @@ -313,9 +313,9 @@ namespace CodexTests private string GetOutputFullPath(TestLifecycle lifecycle, CreateTranscriptAttribute attr) { - var outputPath = Path.GetDirectoryName(lifecycle.Log.LogFile.Filename); + var outputPath = Path.GetDirectoryName(lifecycle.Log.GetFullName()); if (outputPath == null) throw new Exception("Logfile path is null"); - var filename = Path.GetFileNameWithoutExtension(lifecycle.Log.LogFile.Filename); + var filename = Path.GetFileNameWithoutExtension(lifecycle.Log.GetFullName()); if (string.IsNullOrEmpty(filename)) throw new Exception("Logfile name is null or empty"); var outputFile = Path.Combine(outputPath, filename + "_" + attr.OutputFilename); if (!outputFile.EndsWith(".owts")) outputFile += ".owts";