diff --git a/TestCore/DistTest.cs b/TestCore/DistTest.cs index 5cd9348..e7bc7e0 100644 --- a/TestCore/DistTest.cs +++ b/TestCore/DistTest.cs @@ -16,6 +16,7 @@ namespace CodexDistTests.TestCore } else { + TestLog.BeginTest(); fileManager = new FileManager(); k8sManager = new K8sManager(fileManager); } @@ -26,12 +27,13 @@ namespace CodexDistTests.TestCore { try { + TestLog.EndTest(k8sManager); k8sManager.DeleteAllResources(); fileManager.DeleteAllTestFiles(); } catch (Exception ex) { - Console.WriteLine("Cleanup has failed." + ex.Message); + TestLog.Error("Cleanup failed: " + ex.Message); GlobalTestFailure.HasFailed = true; } } diff --git a/TestCore/FileManager.cs b/TestCore/FileManager.cs index 8f6e47f..7caf742 100644 --- a/TestCore/FileManager.cs +++ b/TestCore/FileManager.cs @@ -26,6 +26,7 @@ namespace CodexDistTests.TestCore var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin")); File.Create(result.Filename).Close(); activeFiles.Add(result); + TestLog.Log($"Created test file '{result.Filename}'."); return result; } @@ -33,6 +34,7 @@ namespace CodexDistTests.TestCore { var result = CreateEmptyTestFile(); GenerateFileBytes(result, size); + TestLog.Log($"Generated {size} bytes of content for file '{result.Filename}'."); return result; } diff --git a/TestCore/K8sManager.cs b/TestCore/K8sManager.cs index 238d9ae..2dc8d6f 100644 --- a/TestCore/K8sManager.cs +++ b/TestCore/K8sManager.cs @@ -43,6 +43,7 @@ namespace CodexDistTests.TestCore CreateService(activeNode, client); WaitUntilOnline(activeNode, client); + TestLog.Log($"{activeNode.Describe()} online."); return codexNode; } @@ -56,6 +57,7 @@ namespace CodexDistTests.TestCore var deploymentName = activeNode.Deployment.Name(); BringOffline(activeNode, client); WaitUntilOffline(deploymentName, client); + TestLog.Log($"{activeNode.Describe()} offline."); return activeNode.Origin; } @@ -70,9 +72,22 @@ namespace CodexDistTests.TestCore WaitUntilNamespaceDeleted(client); } + public void FetchAllPodsLogs(Action onLog) + { + var client = CreateClient(); + foreach (var node in activeNodes.Values) + { + var nodeDescription = node.Describe(); + foreach (var podName in node.ActivePodNames) + { + var stream = client.ReadNamespacedPodLog(podName, k8sNamespace); + onLog($"{nodeDescription}:{podName}", stream); + } + } + } + private void BringOffline(ActiveNode activeNode, Kubernetes client) { - DownloadCodexNodeLog(activeNode, client); DeleteDeployment(activeNode, client); DeleteService(activeNode, client); } @@ -270,21 +285,6 @@ namespace CodexDistTests.TestCore return new Kubernetes(config); } - private void DownloadCodexNodeLog(ActiveNode node, Kubernetes client) - { - //var client = CreateClient(); - var i = 0; - foreach (var podName in node.ActivePodNames) - { - var stream = client.ReadNamespacedPodLog(podName, k8sNamespace); - using (var fileStream = File.Create(node.SelectorName + i.ToString() + ".txt")) - { - stream.CopyTo(fileStream); - } - i++; - } - } - private ActiveNode GetAndRemoveActiveNodeFor(IOnlineCodexNode node) { var n = (OnlineCodexNode)node; @@ -357,6 +357,11 @@ namespace CodexDistTests.TestCore { return "codex-test-node"; } + + public string Describe() + { + return $"CodexNode{SelectorName}-Port:{Port}-{Origin.Describe()}"; + } } } } diff --git a/TestCore/OfflineCodexNode.cs b/TestCore/OfflineCodexNode.cs index 1173960..93c6ee2 100644 --- a/TestCore/OfflineCodexNode.cs +++ b/TestCore/OfflineCodexNode.cs @@ -52,5 +52,14 @@ StorageQuota = storageQuotaBytes; return this; } + + public string Describe() + { + var result = ""; + if (LogLevel != null) result += $"LogLevel={LogLevel},"; + if (BootstrapNode != null) result += "BootstrapNode=set,"; + if (StorageQuota != null) result += $"StorageQuote={StorageQuota},"; + return result; + } } } diff --git a/TestCore/TestLog.cs b/TestCore/TestLog.cs new file mode 100644 index 0000000..f21a619 --- /dev/null +++ b/TestCore/TestLog.cs @@ -0,0 +1,126 @@ +using NUnit.Framework; + +namespace CodexDistTests.TestCore +{ + public class TestLog + { + public const string LogRoot = "D:/CodexTestLogs"; + + private static LogFile? file = null; + + public static void Log(string message) + { + file!.Write(message); + } + + public static void Error(string message) + { + Log($"[ERROR] {message}"); + } + + public static void BeginTest() + { + if (file != null) throw new InvalidOperationException("Test is already started!"); + + var name = GetTestName(); + file = new LogFile(name); + + Log($"Begin: {name}"); + } + + public static void EndTest(K8sManager k8sManager) + { + if (file == null) throw new InvalidOperationException("No test is started!"); + + + var result = TestContext.CurrentContext.Result; + + Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + IncludeFullPodLogging(k8sManager); + } + + LogRaw(""); + file = null; + } + + private static string GetTestName() + { + var test = TestContext.CurrentContext.Test; + var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); + return $"{className}.{test.MethodName}"; + } + + private static void LogRaw(string message) + { + file!.WriteRaw(message); + } + + private static void IncludeFullPodLogging(K8sManager k8sManager) + { + LogRaw("Full pod logging:"); + k8sManager.FetchAllPodsLogs(WritePodLog); + } + + private static void WritePodLog(string nodeDescription, Stream stream) + { + LogRaw("---"); + LogRaw(nodeDescription); + var reader = new StreamReader(stream); + var line = reader.ReadLine(); + while (line != null) + { + LogRaw(line); + line = reader.ReadLine(); + } + } + } + + public class LogFile + { + private readonly string filename; + + public LogFile(string name) + { + var now = DateTime.UtcNow; + + var filepath = Path.Join( + TestLog.LogRoot, + $"{now.Year}-{Pad(now.Month)}", + Pad(now.Day)); + + Directory.CreateDirectory(filepath); + + filename = Path.Combine(filepath, + $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log"); + } + + public void Write(string message) + { + WriteRaw($"{GetTimestamp()} {message}"); + } + + public void WriteRaw(string message) + { + try + { + File.AppendAllLines(filename, new[] { message }); + } + catch (Exception ex) + { + Console.WriteLine("Writing to log has failed: " + ex); + } + } + + private static string Pad(int n) + { + return n.ToString().PadLeft(2, '0'); + } + + private static string GetTimestamp() + { + return $"[{DateTime.UtcNow.ToString("u")}]"; + } + } +}