Better logging

CodexDistTestCore/CodexNodeGroup.cs

@@ -2,17 +2,17 @@
 
 namespace CodexDistTestCore
 {
-    public interface IOnlineCodexNodes
+    public interface ICodexNodeGroup
     {
         IOfflineCodexNodes BringOffline();
         IOnlineCodexNode this[int index] { get; }
     }
 
-    public class OnlineCodexNodes : IOnlineCodexNodes
+    public class CodexNodeGroup : ICodexNodeGroup
     {
         private readonly IK8sManager k8SManager;
 
-        public OnlineCodexNodes(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
+        public CodexNodeGroup(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes)
         {
             OrderNumber = orderNumber;
             Origin = origin;
@@ -70,7 +70,7 @@ namespace CodexDistTestCore
 
         public string Describe()
         {
-            return $"CodexNode{OrderNumber}-{Origin.Describe()}";
+            return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}";
         }
     }
 }

CodexDistTestCore/DistTest.cs

@@ -5,6 +5,7 @@ namespace CodexDistTestCore
     [SetUpFixture]
     public abstract class DistTest
     {
+        private TestLog log = null!;
         private FileManager fileManager = null!;
         private K8sManager k8sManager = null!;
 
@@ -13,11 +14,22 @@ namespace CodexDistTestCore
         {
             // Previous test run may have been interrupted.
             // Begin by cleaning everything up.
-            fileManager = new FileManager();
+            log = new TestLog();
-            k8sManager = new K8sManager(fileManager);
+            fileManager = new FileManager(log);
+            k8sManager = new K8sManager(log, fileManager);
 
-            k8sManager.DeleteAllResources();
+            try
-            fileManager.DeleteAllTestFiles();
+            {
+                k8sManager.DeleteAllResources();
+                fileManager.DeleteAllTestFiles();
+            }
+            catch (Exception ex)
+            {
+                GlobalTestFailure.HasFailed = true;
+                log.Error($"Global setup cleanup failed with: {ex}");
+                throw;
+            }
+            log.Log("Global setup cleanup successful");
         }
 
         [SetUp]
@@ -29,9 +41,9 @@ namespace CodexDistTestCore
             }
             else
             {
-                TestLog.BeginTest();
+                log = new TestLog();
-                fileManager = new FileManager();
+                fileManager = new FileManager(log);
-                k8sManager = new K8sManager(fileManager);
+                k8sManager = new K8sManager(log, fileManager);
             }
         }
 
@@ -40,13 +52,13 @@ namespace CodexDistTestCore
         {
             try
             {
-                TestLog.EndTest(k8sManager);
+                log.EndTest(k8sManager);
                 k8sManager.DeleteAllResources();
                 fileManager.DeleteAllTestFiles();
             }
             catch (Exception ex)
             {
-                TestLog.Error("Cleanup failed: " + ex.Message);
+                log.Error("Cleanup failed: " + ex.Message);
                 GlobalTestFailure.HasFailed = true;
             }
         }

CodexDistTestCore/FileManager.cs

@@ -15,10 +15,12 @@ namespace CodexDistTestCore
         private const string Folder = "TestDataFiles";
         private readonly Random random = new Random();
         private readonly List<TestFile> activeFiles = new List<TestFile>();
+        private readonly TestLog log;
 
-        public FileManager()
+        public FileManager(TestLog log)
         {
             if (!Directory.Exists(Folder)) Directory.CreateDirectory(Folder);
+            this.log = log;
         }
 
         public TestFile CreateEmptyTestFile()
@@ -26,7 +28,7 @@ namespace CodexDistTestCore
             var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin"));
             File.Create(result.Filename).Close();
             activeFiles.Add(result);
-            TestLog.Log($"Created test file '{result.Filename}'.");
+            log.Log($"Created test file '{result.Filename}'.");
             return result;
         }
 
@@ -34,7 +36,7 @@ namespace CodexDistTestCore
         {
             var result = CreateEmptyTestFile();
             GenerateFileBytes(result, size);
-            TestLog.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'.");
+            log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'.");
             return result;
         }
 
@@ -73,11 +75,19 @@ namespace CodexDistTestCore
 
         public string Filename { get; }
 
+        public long GetFileSize()
+        {
+            var info = new FileInfo(Filename);
+            return info.Length;
+        }
+
         public void AssertIsEqual(TestFile? other)
         {
             if (other == null) Assert.Fail("TestFile is null.");
             if (other == this || other!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
 
+            if (GetFileSize() != other.GetFileSize()) Assert.Fail("file sizes unequal?");
+
             using var stream1 = new FileStream(Filename, FileMode.Open, FileAccess.Read);
             using var stream2 = new FileStream(other.Filename, FileMode.Open, FileAccess.Read);
 

CodexDistTestCore/K8sManager.cs

@@ -2,40 +2,42 @@
 {
     public interface IK8sManager
     {
-        IOnlineCodexNodes BringOnline(OfflineCodexNodes node);
+        ICodexNodeGroup BringOnline(OfflineCodexNodes node);
-        IOfflineCodexNodes BringOffline(IOnlineCodexNodes node);
+        IOfflineCodexNodes BringOffline(ICodexNodeGroup node);
     }
 
     public class K8sManager : IK8sManager
     {
         private readonly NumberSource onlineCodexNodeOrderNumberSource = new NumberSource(0);
-        private readonly List<OnlineCodexNodes> onlineCodexNodes = new List<OnlineCodexNodes>();
+        private readonly List<CodexNodeGroup> onlineCodexNodes = new List<CodexNodeGroup>();
         private readonly KnownK8sPods knownPods = new KnownK8sPods();
+        private readonly TestLog log;
         private readonly IFileManager fileManager;
 
-        public K8sManager(IFileManager fileManager)
+        public K8sManager(TestLog log, IFileManager fileManager)
         {
+            this.log = log;
             this.fileManager = fileManager;
         }
 
-        public IOnlineCodexNodes BringOnline(OfflineCodexNodes offline)
+        public ICodexNodeGroup BringOnline(OfflineCodexNodes offline)
         {
             var online = CreateOnlineCodexNodes(offline);
 
             K8s().BringOnline(online, offline);
 
-            TestLog.Log($"{offline.NumberOfNodes} Codex nodes online.");
+            log.Log($"{online.Describe()} online.");
 
             return online;
         }
 
-        public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node)
+        public IOfflineCodexNodes BringOffline(ICodexNodeGroup node)
         {
             var online = GetAndRemoveActiveNodeFor(node);
 
             K8s().BringOffline(online);
 
-            TestLog.Log($"{online.Describe()} offline.");
+            log.Log($"{online.Describe()} offline.");
 
             return online.Origin;
         }
@@ -50,11 +52,11 @@
             K8s().FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler);
         }
 
-        private OnlineCodexNodes CreateOnlineCodexNodes(OfflineCodexNodes offline)
+        private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline)
         {
             var containers = CreateContainers(offline.NumberOfNodes);
-            var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray();
+            var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray();
-            var result = new OnlineCodexNodes(onlineCodexNodeOrderNumberSource.GetNextNumber(), offline, this, online);
+            var result = new CodexNodeGroup(onlineCodexNodeOrderNumberSource.GetNextNumber(), offline, this, online);
             onlineCodexNodes.Add(result);
             return result;
         }
@@ -67,9 +69,9 @@
             return containers.ToArray();
         }
 
-        private OnlineCodexNodes GetAndRemoveActiveNodeFor(IOnlineCodexNodes node)
+        private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node)
         {
-            var n = (OnlineCodexNodes)node;
+            var n = (CodexNodeGroup)node;
             onlineCodexNodes.Remove(n);
             return n;
         }

CodexDistTestCore/K8sOperations.cs

@@ -21,7 +21,7 @@ namespace CodexDistTestCore
             client = new Kubernetes(config);
         }
 
-        public void BringOnline(OnlineCodexNodes online, OfflineCodexNodes offline)
+        public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline)
         {
             EnsureTestNamespace();
 
@@ -32,7 +32,7 @@ namespace CodexDistTestCore
             AssignActivePodNames(online);
         }
 
-        public void BringOffline(OnlineCodexNodes online)
+        public void BringOffline(CodexNodeGroup online)
         {
             var deploymentName = online.Deployment.Name();
             DeleteDeployment(online);
@@ -48,7 +48,7 @@ namespace CodexDistTestCore
             WaitUntilNamespaceDeleted();
         }
 
-        public void FetchAllPodsLogs(OnlineCodexNodes[] onlines, IPodLogsHandler logHandler)
+        public void FetchAllPodsLogs(CodexNodeGroup[] onlines, IPodLogsHandler logHandler)
         {
             foreach (var online in onlines)
             {
@@ -61,7 +61,7 @@ namespace CodexDistTestCore
             }
         }
 
-        private void AssignActivePodNames(OnlineCodexNodes online)
+        private void AssignActivePodNames(CodexNodeGroup online)
         {
             var pods = client.ListNamespacedPod(K8sNamespace);
             var podNames = pods.Items.Select(p => p.Name());
@@ -77,7 +77,7 @@ namespace CodexDistTestCore
 
         #region Waiting
 
-        private void WaitUntilOnline(OnlineCodexNodes online)
+        private void WaitUntilOnline(CodexNodeGroup online)
         {
             WaitUntil(() =>
             {
@@ -126,7 +126,7 @@ namespace CodexDistTestCore
 
         #region Service management
 
-        private void CreateService(OnlineCodexNodes online)
+        private void CreateService(CodexNodeGroup online)
         {
             var serviceSpec = new V1Service
             {
@@ -143,7 +143,7 @@ namespace CodexDistTestCore
             online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
         }
 
-        private List<V1ServicePort> CreateServicePorts(OnlineCodexNodes online)
+        private List<V1ServicePort> CreateServicePorts(CodexNodeGroup online)
         {
             var result = new List<V1ServicePort>();
             var containers = online.GetContainers();
@@ -160,7 +160,7 @@ namespace CodexDistTestCore
             return result;
         }
 
-        private void DeleteService(OnlineCodexNodes online)
+        private void DeleteService(CodexNodeGroup online)
         {
             if (online.Service == null) return;
             client.DeleteNamespacedService(online.Service.Name(), K8sNamespace);
@@ -171,7 +171,7 @@ namespace CodexDistTestCore
 
         #region Deployment management
 
-        private void CreateDeployment(OnlineCodexNodes online, OfflineCodexNodes offline)
+        private void CreateDeployment(CodexNodeGroup online, OfflineCodexNodes offline)
         {
             var deploymentSpec = new V1Deployment
             {
@@ -201,7 +201,7 @@ namespace CodexDistTestCore
             online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
         }
 
-        private List<V1Container> CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline)
+        private List<V1Container> CreateDeploymentContainers(CodexNodeGroup online, OfflineCodexNodes offline)
         {
             var result = new List<V1Container>();
             var containers = online.GetContainers();
@@ -225,7 +225,7 @@ namespace CodexDistTestCore
             return result;
         }
 
-        private void DeleteDeployment(OnlineCodexNodes online)
+        private void DeleteDeployment(CodexNodeGroup online)
         {
             if (online.Deployment == null) return;
             client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace);

CodexDistTestCore/OfflineCodexNodes.cs

@@ -5,7 +5,7 @@
         IOfflineCodexNodes WithLogLevel(CodexLogLevel level);
         IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node);
         IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota);
-        IOnlineCodexNodes BringOnline();
+        ICodexNodeGroup BringOnline();
     }
 
     public enum CodexLogLevel
@@ -32,7 +32,7 @@
             NumberOfNodes = numberOfNodes;
         }
 
-        public IOnlineCodexNodes BringOnline()
+        public ICodexNodeGroup BringOnline()
         {
             return k8SManager.BringOnline(this);
         }
@@ -57,11 +57,15 @@
 
         public string Describe()
         {
-            var result = "";
+            var args = string.Join(',', DescribeArgs());
-            if (LogLevel != null) result += $"LogLevel={LogLevel},";
+            return $"{NumberOfNodes} CodexNodes with [{args}]";
-            if (BootstrapNode != null) result += "BootstrapNode=set,";
+        }
-            if (StorageQuota != null) result += $"StorageQuote={StorageQuota.SizeInBytes},";
+
-            return result;
+        private IEnumerable<string> DescribeArgs()
+        {
+            if (LogLevel != null) yield return ($"LogLevel={LogLevel}");
+            if (BootstrapNode != null) yield return ("BootstrapNode=set");
+            if (StorageQuota != null) yield return ($"StorageQuote={StorageQuota.SizeInBytes}");
         }
     }
 }

CodexDistTestCore/OnlineCodexNode.cs

@@ -11,38 +11,46 @@ namespace CodexDistTestCore
 
     public class OnlineCodexNode : IOnlineCodexNode
     {
+        private readonly TestLog log;
         private readonly IFileManager fileManager;
 
-        public OnlineCodexNode(IFileManager fileManager, CodexNodeContainer environment)
+        public OnlineCodexNode(TestLog log, IFileManager fileManager, CodexNodeContainer container)
         {
+            this.log = log;
             this.fileManager = fileManager;
-            Container = environment;
+            Container = container;
         }
 
         public CodexNodeContainer Container { get; }
 
         public CodexDebugResponse GetDebugInfo()
         {
-            return Http().HttpGetJson<CodexDebugResponse>("debug/info");
+            var response = Http().HttpGetJson<CodexDebugResponse>("debug/info");
+            Log("Got DebugInfo with id: " + response.id);
+            return response;
         }
 
         public ContentId UploadFile(TestFile file)
         {
+            Log($"Uploading file of size {file.GetFileSize()}");
             using var fileStream = File.OpenRead(file.Filename);
             var response = Http().HttpPostStream("upload", fileStream);
             if (response.StartsWith("Unable to store block"))
             {
                 Assert.Fail("Node failed to store block.");
             }
+            Log($"Uploaded file. Received contentId: {response}");
             return new ContentId(response);
         }
 
         public TestFile? DownloadContent(ContentId contentId)
         {
+            Log($"Downloading for contentId: {contentId}");
             var file = fileManager.CreateEmptyTestFile();
             using var fileStream = File.OpenWrite(file.Filename);
             using var downloadStream = Http().HttpGetStream("download/" + contentId.Id);
             downloadStream.CopyTo(fileStream);
+            Log($"Downloaded file of size {file.GetFileSize()}");
             return file;
         }
 
@@ -50,6 +58,11 @@ namespace CodexDistTestCore
         {
             return new Http(ip: "127.0.0.1", port: Container.ServicePort, baseUrl: "/api/codex/v1");
         }
+
+        private void Log(string msg)
+        {
+            log.Log($"Node {Container.Name}: {msg}");
+        }
     }
 
     public class ContentId

CodexDistTestCore/TestLog.cs

@@ -5,35 +5,28 @@ namespace CodexDistTestCore
     public class TestLog
     {
         public const string LogRoot = "D:/CodexTestLogs";
+        private readonly LogFile file;
 
-        private static LogFile? file = null;
+        public TestLog()
-
-        // This is all way too static. It needs to be cleaned up.
-        public static void Log(string message)
         {
-            file!.Write(message);
-        }
-
-        public static void Error(string message)
-        {
-            Log($"[ERROR] {message}");
-        }
-
-        public static void BeginTest()
-        {
-            if (file != null) throw new InvalidOperationException("Test is already started!");
-
             var name = GetTestName();
             file = new LogFile(name);
 
             Log($"Begin: {name}");
         }
 
-        public static void EndTest(K8sManager k8sManager)
+        public void Log(string message)
         {
-            if (file == null) throw new InvalidOperationException("No test is started!");
+            file.Write(message);
+        }
 
+        public void Error(string message)
+        {
+            Log($"[ERROR] {message}");
+        }
 
+        public void EndTest(K8sManager k8sManager)
+        {
             var result = TestContext.CurrentContext.Result;
 
             Log($"Finished: {GetTestName()} = {result.Outcome.Status}");
@@ -42,8 +35,6 @@ namespace CodexDistTestCore
                 var logWriter = new PodLogWriter(file);
                 logWriter.IncludeFullPodLogging(k8sManager);
             }
-
-            file = null;
         }
 
         private static string GetTestName()
@@ -65,14 +56,14 @@ namespace CodexDistTestCore
 
         public void IncludeFullPodLogging(K8sManager k8sManager)
         {
-            TestLog.Log("Full pod logging:");
+            file.Write("Full pod logging:");
             k8sManager.FetchAllPodsLogs(this);
         }
 
         public void Log(int id, string podDescription, Stream log)
         {
             var logFile = id.ToString().PadLeft(6, '0');
-            TestLog.Log($"{podDescription} -->> {logFile}");
+            file.Write($"{podDescription} -->> {logFile}");
             LogRaw(podDescription, logFile);
             var reader = new StreamReader(log);
             var line = reader.ReadLine();