From ce963a228caa9fb039dffecde818839228d17773 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 26 Mar 2023 10:45:01 +0200 Subject: [PATCH] Successful access to codex log from within test --- CodexDistTestCore/CodexNodeGroup.cs | 11 ++- CodexDistTestCore/CodexNodeLog.cs | 29 +++++++ CodexDistTestCore/DistTest.cs | 22 ++++- CodexDistTestCore/K8sManager.cs | 18 ++-- CodexDistTestCore/K8sOperations.cs | 21 +---- CodexDistTestCore/OnlineCodexNode.cs | 11 +++ CodexDistTestCore/PodLogDownloader.cs | 60 ++++++++++++++ CodexDistTestCore/PodLogsHandler.cs | 7 -- CodexDistTestCore/TestLog.cs | 51 +----------- Tests/BasicTests/SimpleTests.cs | 113 ++++++++++++++------------ 10 files changed, 211 insertions(+), 132 deletions(-) create mode 100644 CodexDistTestCore/CodexNodeLog.cs create mode 100644 CodexDistTestCore/PodLogDownloader.cs delete mode 100644 CodexDistTestCore/PodLogsHandler.cs diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs index eb52eca..975eccf 100644 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -12,10 +12,12 @@ namespace CodexDistTestCore public class CodexNodeGroup : ICodexNodeGroup { + private readonly TestLog log; private readonly IK8sManager k8SManager; - public CodexNodeGroup(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes) + public CodexNodeGroup(TestLog log, int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes) { + this.log = log; OrderNumber = orderNumber; Origin = origin; this.k8SManager = k8SManager; @@ -77,6 +79,13 @@ namespace CodexDistTestCore }; } + public CodexNodeLog DownloadLog(IOnlineCodexNode node) + { + var logDownloader = new PodLogDownloader(log, k8SManager); + var n = (OnlineCodexNode)node; + return logDownloader.DownloadLog(n); + } + public Dictionary GetSelector() { return new Dictionary { { "codex-test-node", "dist-test-" + OrderNumber } }; diff --git a/CodexDistTestCore/CodexNodeLog.cs b/CodexDistTestCore/CodexNodeLog.cs new file mode 100644 index 0000000..b4cae47 --- /dev/null +++ b/CodexDistTestCore/CodexNodeLog.cs @@ -0,0 +1,29 @@ +using NUnit.Framework; + +namespace CodexDistTestCore +{ + public class CodexNodeLog + { + private readonly LogFile logFile; + + public CodexNodeLog(LogFile logFile) + { + this.logFile = logFile; + } + + public void AssertLogContains(string expectedString) + { + using var file = File.OpenRead(logFile.FullFilename); + using var streamReader = new StreamReader(file); + + var line = streamReader.ReadLine(); + while (line != null) + { + if (line.Contains(expectedString)) return; + line = streamReader.ReadLine(); + } + + Assert.Fail($"Unable to find string '{expectedString}' in CodexNode log file {logFile.FilenameWithoutPath}"); + } + } +} diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index db32997..eecc4eb 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -56,7 +56,8 @@ namespace CodexDistTestCore { try { - log.EndTest(k8sManager); + log.EndTest(); + IncludeLogsOnTestFailure(); k8sManager.DeleteAllResources(); fileManager.DeleteAllTestFiles(); } @@ -76,6 +77,25 @@ namespace CodexDistTestCore { return new OfflineCodexNodes(k8sManager, numberOfNodes); } + + private void IncludeLogsOnTestFailure() + { + var result = TestContext.CurrentContext.Result; + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + k8sManager.ForEachOnlineGroup(DownloadLogs); + } + } + + private void DownloadLogs(CodexNodeGroup group) + { + foreach (var node in group) + { + var downloader = new PodLogDownloader(log, k8sManager); + var n = (OnlineCodexNode)node; + downloader.DownloadLog(n); + } + } } public static class GlobalTestFailure diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 3530c96..22ec144 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -4,12 +4,13 @@ { ICodexNodeGroup BringOnline(OfflineCodexNodes node); IOfflineCodexNodes BringOffline(ICodexNodeGroup node); + void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler); } public class K8sManager : IK8sManager { private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource(); - private readonly List onlineCodexNodes = new List(); + private readonly List onlineCodexNodeGroups = new List(); private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly TestLog log; private readonly IFileManager fileManager; @@ -47,17 +48,22 @@ K8s(k => k.DeleteAllResources()); } - public void FetchAllPodsLogs(IPodLogsHandler logHandler) + public void ForEachOnlineGroup(Action action) { - K8s(k => k.FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler)); + foreach (var group in onlineCodexNodeGroups) action(group); + } + + public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler) + { + K8s(k => k.FetchPodLog(node, logHandler)); } private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) { var containers = CreateContainers(offline.NumberOfNodes); var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray(); - var result = new CodexNodeGroup(codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online); - onlineCodexNodes.Add(result); + var result = new CodexNodeGroup(log, codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online); + onlineCodexNodeGroups.Add(result); return result; } @@ -72,7 +78,7 @@ private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node) { var n = (CodexNodeGroup)node; - onlineCodexNodes.Remove(n); + onlineCodexNodeGroups.Remove(n); return n; } diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 0febe44..a9f1613 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -51,25 +51,10 @@ namespace CodexDistTestCore WaitUntilNamespaceDeleted(); } - public void FetchAllPodsLogs(CodexNodeGroup[] onlines, IPodLogsHandler logHandler) + public void FetchPodLog(OnlineCodexNode node, IPodLogHandler logHandler) { - var logNumberSource = new NumberSource(0); - foreach (var online in onlines) - { - foreach (var node in online) - { - WritePodLogs(online, node, logHandler, logNumberSource); - } - } - } - - private void WritePodLogs(CodexNodeGroup online, IOnlineCodexNode node, IPodLogsHandler logHandler, NumberSource logNumberSource) - { - var n = (OnlineCodexNode)node; - var nodeDescription = $"{online.Describe()} contains {n.GetName()}"; - - var stream = client.ReadNamespacedPodLog(online.PodInfo!.Name, K8sNamespace, n.Container.Name); - logHandler.Log(logNumberSource.GetNextNumber(), nodeDescription, stream); + var stream = client.ReadNamespacedPodLog(node.Group.PodInfo!.Name, K8sNamespace, node.Container.Name); + logHandler.Log(stream); } private void FetchPodInfo(CodexNodeGroup online) diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index 6b523bc..f8cf90f 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -9,6 +9,7 @@ namespace CodexDistTestCore ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); void ConnectToPeer(IOnlineCodexNode node); + CodexNodeLog DownloadLog(); } public class OnlineCodexNode : IOnlineCodexNode @@ -79,6 +80,16 @@ namespace CodexDistTestCore Log($"Successfully connected to peer {peer.GetName()}."); } + public CodexNodeLog DownloadLog() + { + return Group.DownloadLog(this); + } + + public string Describe() + { + return $"{Group.Describe()} contains {GetName()}"; + } + private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) { var multiAddress = peerInfo.addrs.First(); diff --git a/CodexDistTestCore/PodLogDownloader.cs b/CodexDistTestCore/PodLogDownloader.cs new file mode 100644 index 0000000..b6a2ed7 --- /dev/null +++ b/CodexDistTestCore/PodLogDownloader.cs @@ -0,0 +1,60 @@ +namespace CodexDistTestCore +{ + public interface IPodLogHandler + { + void Log(Stream log); + } + + public class PodLogDownloader + { + private readonly TestLog log; + private readonly IK8sManager k8SManager; + + public PodLogDownloader(TestLog log, IK8sManager k8sManager) + { + this.log = log; + k8SManager = k8sManager; + } + + public CodexNodeLog DownloadLog(OnlineCodexNode node) + { + var description = node.Describe(); + var subFile = log.CreateSubfile(); + + log.Log($"Downloading logs for {description} to file {subFile.FilenameWithoutPath}"); + var handler = new PodLogDownloadHandler(description, subFile); + k8SManager.FetchPodLog(node, handler); + return handler.CreateCodexNodeLog(); + } + } + + public class PodLogDownloadHandler : IPodLogHandler + { + private readonly string description; + private readonly LogFile log; + + public PodLogDownloadHandler(string description, LogFile log) + { + this.description = description; + this.log = log; + } + + public CodexNodeLog CreateCodexNodeLog() + { + return new CodexNodeLog(log); + } + + public void Log(Stream stream) + { + log.Write($"{description} -->> {log.FilenameWithoutPath}"); + log.WriteRaw(description); + var reader = new StreamReader(stream); + var line = reader.ReadLine(); + while (line != null) + { + log.WriteRaw(line); + line = reader.ReadLine(); + } + } + } +} diff --git a/CodexDistTestCore/PodLogsHandler.cs b/CodexDistTestCore/PodLogsHandler.cs deleted file mode 100644 index d7898f5..0000000 --- a/CodexDistTestCore/PodLogsHandler.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace CodexDistTestCore -{ - public interface IPodLogsHandler - { - void Log(int id, string podDescription, Stream log); - } -} diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index f73873b..aca6f1b 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -29,23 +29,15 @@ namespace CodexDistTestCore Log($"[ERROR] {message}"); } - public void EndTest(K8sManager k8sManager) + public void EndTest() { var result = TestContext.CurrentContext.Result; Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); - if (!string.IsNullOrEmpty(result.Message)) { Log(result.Message); - } - - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) - { Log($"{result.StackTrace}"); - - var logWriter = new PodLogWriter(CreateSubfile()); - logWriter.IncludeFullPodLogging(k8sManager); } } @@ -69,44 +61,9 @@ namespace CodexDistTestCore } } - public class PodLogWriter : IPodLogsHandler - { - private readonly LogFile file; - - public PodLogWriter(LogFile file) - { - this.file = file; - } - - public void IncludeFullPodLogging(K8sManager k8sManager) - { - file.Write("Full pod logging:"); - k8sManager.FetchAllPodsLogs(this); - } - - public void Log(int id, string podDescription, Stream log) - { - file.Write($"{podDescription} -->> {file.FilenameWithoutPath}"); - LogRaw(podDescription); - var reader = new StreamReader(log); - var line = reader.ReadLine(); - while (line != null) - { - LogRaw(line); - line = reader.ReadLine(); - } - } - - private void LogRaw(string message) - { - file!.WriteRaw(message); - } - } - public class LogFile { private readonly string filepath; - private readonly string filename; public LogFile(DateTime now, string name) { @@ -118,10 +75,10 @@ namespace CodexDistTestCore Directory.CreateDirectory(filepath); FilenameWithoutPath = $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log"; - - filename = Path.Combine(filepath, FilenameWithoutPath); + FullFilename = Path.Combine(filepath, FilenameWithoutPath); } + public string FullFilename { get; } public string FilenameWithoutPath { get; } public void Write(string message) @@ -133,7 +90,7 @@ namespace CodexDistTestCore { try { - File.AppendAllLines(filename, new[] { message }); + File.AppendAllLines(FullFilename, new[] { message }); } catch (Exception ex) { diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index a56b09d..3a072e6 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -1,5 +1,4 @@ using CodexDistTestCore; -using CodexDistTestCore.Config; using NUnit.Framework; namespace Tests.BasicTests @@ -8,77 +7,87 @@ namespace Tests.BasicTests public class SimpleTests : DistTest { [Test] - public void GetDebugInfo() + public void CanAccessLogs() { - var dockerImage = new CodexDockerImage(); + var group = SetupCodexNodes(2).BringOnline(); - var node = SetupCodexNodes(1) - .BringOnline()[0]; + var log = group[0].DownloadLog(); - var debugInfo = node.GetDebugInfo(); - - Assert.That(debugInfo.spr, Is.Not.Empty); - Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision())); + log.AssertLogContains("topics=\"discv5\""); } - [Test] - public void OneClientTest() - { - var primary = SetupCodexNodes(1) - .BringOnline()[0]; + //[Test] + //public void GetDebugInfo() + //{ + // var dockerImage = new CodexDockerImage(); - var testFile = GenerateTestFile(1.MB()); + // var node = SetupCodexNodes(1) + // .BringOnline()[0]; - var contentId = primary.UploadFile(testFile); + // var debugInfo = node.GetDebugInfo(); - var downloadedFile = primary.DownloadContent(contentId); + // Assert.That(debugInfo.spr, Is.Not.Empty); + // Assert.That(debugInfo.codex.revision, Is.EqualTo(dockerImage.GetExpectedImageRevision())); + //} - testFile.AssertIsEqual(downloadedFile); - } + //[Test] + //public void OneClientTest() + //{ + // var primary = SetupCodexNodes(1) + // .BringOnline()[0]; - [Test] - public void TwoClientsOnePodTest() - { - var group = SetupCodexNodes(2) - .BringOnline(); + // var testFile = GenerateTestFile(1.MB()); - var primary = group[0]; - var secondary = group[1]; + // var contentId = primary.UploadFile(testFile); - PerformTwoClientTest(primary, secondary); - } + // var downloadedFile = primary.DownloadContent(contentId); - [Test] - public void TwoClientsTwoPodsTest() - { - var primary = SetupCodexNodes(1) - .BringOnline()[0]; + // testFile.AssertIsEqual(downloadedFile); + //} - var secondary = SetupCodexNodes(1) - .BringOnline()[0]; + //[Test] + //public void TwoClientsOnePodTest() + //{ + // var group = SetupCodexNodes(2) + // .BringOnline(); - PerformTwoClientTest(primary, secondary); - } + // var primary = group[0]; + // var secondary = group[1]; - [Test] - public void TwoClientsTwoLocationsTest() - { - var primary = SetupCodexNodes(1) - .At(Location.BensLaptop) - .BringOnline()[0]; + // PerformTwoClientTest(primary, secondary); + //} - var secondary = SetupCodexNodes(1) - .At(Location.BensOldGamingMachine) - .BringOnline()[0]; + //[Test] + //public void TwoClientsTwoPodsTest() + //{ + // var primary = SetupCodexNodes(1) + // .BringOnline()[0]; - var debugInfo = primary.GetDebugInfo(); - Assert.That(debugInfo.spr, Is.Not.Empty); + // var secondary = SetupCodexNodes(1) + // .BringOnline()[0]; - var debugInfo2 = secondary.GetDebugInfo(); - Assert.That(debugInfo2.spr, Is.Not.Empty); + // PerformTwoClientTest(primary, secondary); + //} - PerformTwoClientTest(primary, secondary); - } + //[Test] + //public void TwoClientsTwoLocationsTest() + //{ + // var primary = SetupCodexNodes(1) + // .At(Location.BensLaptop) + // .BringOnline()[0]; + + // var secondary = SetupCodexNodes(1) + // .At(Location.BensOldGamingMachine) + // .BringOnline()[0]; + + // var debugInfo = primary.GetDebugInfo(); + // Assert.That(debugInfo.spr, Is.Not.Empty); + + // var debugInfo2 = secondary.GetDebugInfo(); + // Assert.That(debugInfo2.spr, Is.Not.Empty); + + // PerformTwoClientTest(primary, secondary); + //} private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) {