Adds some serious logging

TestCore/DistTest.cs

@@ -16,6 +16,7 @@ namespace CodexDistTests.TestCore
             }
             else
             {
+                TestLog.BeginTest();
                 fileManager = new FileManager();
                 k8sManager = new K8sManager(fileManager);
             }
@@ -26,12 +27,13 @@ namespace CodexDistTests.TestCore
         {
             try
             {
+                TestLog.EndTest(k8sManager);
                 k8sManager.DeleteAllResources();
                 fileManager.DeleteAllTestFiles();
             }
             catch (Exception ex)
             {
-                Console.WriteLine("Cleanup has failed." + ex.Message);
+                TestLog.Error("Cleanup failed: " + ex.Message);
                 GlobalTestFailure.HasFailed = true;
             }
         }

TestCore/FileManager.cs

@@ -26,6 +26,7 @@ namespace CodexDistTests.TestCore
             var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin"));
             File.Create(result.Filename).Close();
             activeFiles.Add(result);
+            TestLog.Log($"Created test file '{result.Filename}'.");
             return result;
         }
 
@@ -33,6 +34,7 @@ namespace CodexDistTests.TestCore
         {
             var result = CreateEmptyTestFile();
             GenerateFileBytes(result, size);
+            TestLog.Log($"Generated {size} bytes of content for file '{result.Filename}'.");
             return result;
         }
 

TestCore/K8sManager.cs

@@ -43,6 +43,7 @@ namespace CodexDistTests.TestCore
             CreateService(activeNode, client);
 
             WaitUntilOnline(activeNode, client);
+            TestLog.Log($"{activeNode.Describe()} online.");
 
             return codexNode;
         }
@@ -56,6 +57,7 @@ namespace CodexDistTests.TestCore
             var deploymentName = activeNode.Deployment.Name();
             BringOffline(activeNode, client);
             WaitUntilOffline(deploymentName, client);
+            TestLog.Log($"{activeNode.Describe()} offline.");
 
             return activeNode.Origin;
         }
@@ -70,9 +72,22 @@ namespace CodexDistTests.TestCore
             WaitUntilNamespaceDeleted(client);
         }
 
+        public void FetchAllPodsLogs(Action<string, Stream> onLog)
+        {
+            var client = CreateClient();
+            foreach (var node in activeNodes.Values)
+            {
+                var nodeDescription = node.Describe();
+                foreach (var podName in node.ActivePodNames)
+                {
+                    var stream = client.ReadNamespacedPodLog(podName, k8sNamespace);
+                    onLog($"{nodeDescription}:{podName}", stream);
+                }
+            }
+        }
+
         private void BringOffline(ActiveNode activeNode, Kubernetes client)
         {
-            DownloadCodexNodeLog(activeNode, client);
             DeleteDeployment(activeNode, client);
             DeleteService(activeNode, client);
         }
@@ -270,21 +285,6 @@ namespace CodexDistTests.TestCore
             return new Kubernetes(config);
         }
 
-        private void DownloadCodexNodeLog(ActiveNode node, Kubernetes client)
-        {
-            //var client = CreateClient();
-            var i = 0;
-            foreach (var podName in node.ActivePodNames)
-            {
-                var stream = client.ReadNamespacedPodLog(podName, k8sNamespace);
-                using (var fileStream = File.Create(node.SelectorName + i.ToString() + ".txt"))
-                {
-                    stream.CopyTo(fileStream);
-                }
-                i++;
-            }
-        }
-
         private ActiveNode GetAndRemoveActiveNodeFor(IOnlineCodexNode node)
         {
             var n = (OnlineCodexNode)node;
@@ -357,6 +357,11 @@ namespace CodexDistTests.TestCore
             {
                 return "codex-test-node";
             }
+
+            public string Describe()
+            {
+                return $"CodexNode{SelectorName}-Port:{Port}-{Origin.Describe()}";
+            }
         }
     }
 }

TestCore/OfflineCodexNode.cs

@@ -52,5 +52,14 @@
             StorageQuota = storageQuotaBytes;
             return this;
         }
+
+        public string Describe()
+        {
+            var result = "";
+            if (LogLevel != null) result += $"LogLevel={LogLevel},";
+            if (BootstrapNode != null) result += "BootstrapNode=set,";
+            if (StorageQuota != null) result += $"StorageQuote={StorageQuota},";
+            return result;
+        }
     }
 }

TestCore/TestLog.cs

@@ -0,0 +1,126 @@
+using NUnit.Framework;
+
+namespace CodexDistTests.TestCore
+{
+    public class TestLog
+    {
+        public const string LogRoot = "D:/CodexTestLogs";
+
+        private static LogFile? file = null;
+
+        public static void Log(string message)
+        {
+            file!.Write(message);
+        }
+
+        public static void Error(string message) 
+        {
+            Log($"[ERROR] {message}");
+        }
+
+        public static void BeginTest()
+        {
+            if (file != null) throw new InvalidOperationException("Test is already started!");
+
+            var name = GetTestName();
+            file = new LogFile(name);
+
+            Log($"Begin: {name}");
+        }
+
+        public static void EndTest(K8sManager k8sManager)
+        {
+            if (file == null) throw new InvalidOperationException("No test is started!");
+
+            
+            var result = TestContext.CurrentContext.Result;
+
+            Log($"Finished: {GetTestName()} = {result.Outcome.Status}");
+            if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
+            {
+                IncludeFullPodLogging(k8sManager);
+            }
+
+            LogRaw("");
+            file = null;
+        }
+
+        private static string GetTestName()
+        {
+            var test = TestContext.CurrentContext.Test;
+            var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1);
+            return $"{className}.{test.MethodName}";
+        }
+
+        private static void LogRaw(string message)
+        {
+            file!.WriteRaw(message);
+        }
+
+        private static void IncludeFullPodLogging(K8sManager k8sManager)
+        {
+            LogRaw("Full pod logging:");
+            k8sManager.FetchAllPodsLogs(WritePodLog);
+        }
+
+        private static void WritePodLog(string nodeDescription, Stream stream)
+        {
+            LogRaw("---");
+            LogRaw(nodeDescription);
+            var reader = new StreamReader(stream);
+            var line = reader.ReadLine();
+            while (line != null)
+            {
+                LogRaw(line);
+                line = reader.ReadLine();
+            }
+        }
+    }
+
+    public class LogFile
+    {
+        private readonly string filename;
+
+        public LogFile(string name)
+        {
+            var now = DateTime.UtcNow;
+
+            var filepath = Path.Join(
+                TestLog.LogRoot,
+                $"{now.Year}-{Pad(now.Month)}",
+                Pad(now.Day));
+
+            Directory.CreateDirectory(filepath);
+
+            filename = Path.Combine(filepath,
+                $"{Pad(now.Hour)}-{Pad(now.Minute)}-{Pad(now.Second)}Z_{name.Replace('.', '-')}.log");
+        }
+
+        public void Write(string message)
+        {
+            WriteRaw($"{GetTimestamp()} {message}");
+        }
+
+        public void WriteRaw(string message)
+        {
+            try
+            {
+                File.AppendAllLines(filename, new[] {  message });
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine("Writing to log has failed: " + ex);
+            }
+        }
+
+        private static string Pad(int n)
+        {
+            return n.ToString().PadLeft(2, '0');
+        }
+
+        private static string GetTimestamp()
+        {
+            return $"[{DateTime.UtcNow.ToString("u")}]";
+        }
+    }
+}