diff --git a/CodexDistTestCore/OnlineCodexNodes.cs b/CodexDistTestCore/CodexNodeGroup.cs similarity index 86% rename from CodexDistTestCore/OnlineCodexNodes.cs rename to CodexDistTestCore/CodexNodeGroup.cs index c4007fa..23189e5 100644 --- a/CodexDistTestCore/OnlineCodexNodes.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -2,17 +2,17 @@ namespace CodexDistTestCore { - public interface IOnlineCodexNodes + public interface ICodexNodeGroup { IOfflineCodexNodes BringOffline(); IOnlineCodexNode this[int index] { get; } } - public class OnlineCodexNodes : IOnlineCodexNodes + public class CodexNodeGroup : ICodexNodeGroup { private readonly IK8sManager k8SManager; - public OnlineCodexNodes(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes) + public CodexNodeGroup(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes) { OrderNumber = orderNumber; Origin = origin; @@ -70,7 +70,7 @@ namespace CodexDistTestCore public string Describe() { - return $"CodexNode{OrderNumber}-{Origin.Describe()}"; + return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}"; } } } diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index 3e8ea98..55bd75c 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -5,6 +5,7 @@ namespace CodexDistTestCore [SetUpFixture] public abstract class DistTest { + private TestLog log = null!; private FileManager fileManager = null!; private K8sManager k8sManager = null!; @@ -13,11 +14,22 @@ namespace CodexDistTestCore { // Previous test run may have been interrupted. // Begin by cleaning everything up. - fileManager = new FileManager(); - k8sManager = new K8sManager(fileManager); + log = new TestLog(); + fileManager = new FileManager(log); + k8sManager = new K8sManager(log, fileManager); - k8sManager.DeleteAllResources(); - fileManager.DeleteAllTestFiles(); + try + { + k8sManager.DeleteAllResources(); + fileManager.DeleteAllTestFiles(); + } + catch (Exception ex) + { + GlobalTestFailure.HasFailed = true; + log.Error($"Global setup cleanup failed with: {ex}"); + throw; + } + log.Log("Global setup cleanup successful"); } [SetUp] @@ -29,9 +41,9 @@ namespace CodexDistTestCore } else { - TestLog.BeginTest(); - fileManager = new FileManager(); - k8sManager = new K8sManager(fileManager); + log = new TestLog(); + fileManager = new FileManager(log); + k8sManager = new K8sManager(log, fileManager); } } @@ -40,13 +52,13 @@ namespace CodexDistTestCore { try { - TestLog.EndTest(k8sManager); + log.EndTest(k8sManager); k8sManager.DeleteAllResources(); fileManager.DeleteAllTestFiles(); } catch (Exception ex) { - TestLog.Error("Cleanup failed: " + ex.Message); + log.Error("Cleanup failed: " + ex.Message); GlobalTestFailure.HasFailed = true; } } diff --git a/CodexDistTestCore/FileManager.cs b/CodexDistTestCore/FileManager.cs index a040e09..94abe7a 100644 --- a/CodexDistTestCore/FileManager.cs +++ b/CodexDistTestCore/FileManager.cs @@ -15,10 +15,12 @@ namespace CodexDistTestCore private const string Folder = "TestDataFiles"; private readonly Random random = new Random(); private readonly List activeFiles = new List(); + private readonly TestLog log; - public FileManager() + public FileManager(TestLog log) { if (!Directory.Exists(Folder)) Directory.CreateDirectory(Folder); + this.log = log; } public TestFile CreateEmptyTestFile() @@ -26,7 +28,7 @@ namespace CodexDistTestCore var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin")); File.Create(result.Filename).Close(); activeFiles.Add(result); - TestLog.Log($"Created test file '{result.Filename}'."); + log.Log($"Created test file '{result.Filename}'."); return result; } @@ -34,7 +36,7 @@ namespace CodexDistTestCore { var result = CreateEmptyTestFile(); GenerateFileBytes(result, size); - TestLog.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'."); + log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'."); return result; } @@ -73,11 +75,19 @@ namespace CodexDistTestCore public string Filename { get; } + public long GetFileSize() + { + var info = new FileInfo(Filename); + return info.Length; + } + public void AssertIsEqual(TestFile? other) { if (other == null) Assert.Fail("TestFile is null."); if (other == this || other!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); + if (GetFileSize() != other.GetFileSize()) Assert.Fail("file sizes unequal?"); + using var stream1 = new FileStream(Filename, FileMode.Open, FileAccess.Read); using var stream2 = new FileStream(other.Filename, FileMode.Open, FileAccess.Read); diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 45c6b42..cc15a5d 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -2,40 +2,42 @@ { public interface IK8sManager { - IOnlineCodexNodes BringOnline(OfflineCodexNodes node); - IOfflineCodexNodes BringOffline(IOnlineCodexNodes node); + ICodexNodeGroup BringOnline(OfflineCodexNodes node); + IOfflineCodexNodes BringOffline(ICodexNodeGroup node); } public class K8sManager : IK8sManager { private readonly NumberSource onlineCodexNodeOrderNumberSource = new NumberSource(0); - private readonly List onlineCodexNodes = new List(); + private readonly List onlineCodexNodes = new List(); private readonly KnownK8sPods knownPods = new KnownK8sPods(); + private readonly TestLog log; private readonly IFileManager fileManager; - public K8sManager(IFileManager fileManager) + public K8sManager(TestLog log, IFileManager fileManager) { + this.log = log; this.fileManager = fileManager; } - public IOnlineCodexNodes BringOnline(OfflineCodexNodes offline) + public ICodexNodeGroup BringOnline(OfflineCodexNodes offline) { var online = CreateOnlineCodexNodes(offline); K8s().BringOnline(online, offline); - TestLog.Log($"{offline.NumberOfNodes} Codex nodes online."); + log.Log($"{online.Describe()} online."); return online; } - public IOfflineCodexNodes BringOffline(IOnlineCodexNodes node) + public IOfflineCodexNodes BringOffline(ICodexNodeGroup node) { var online = GetAndRemoveActiveNodeFor(node); K8s().BringOffline(online); - TestLog.Log($"{online.Describe()} offline."); + log.Log($"{online.Describe()} offline."); return online.Origin; } @@ -50,11 +52,11 @@ K8s().FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler); } - private OnlineCodexNodes CreateOnlineCodexNodes(OfflineCodexNodes offline) + private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) { var containers = CreateContainers(offline.NumberOfNodes); - var online = containers.Select(c => new OnlineCodexNode(fileManager, c)).ToArray(); - var result = new OnlineCodexNodes(onlineCodexNodeOrderNumberSource.GetNextNumber(), offline, this, online); + var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray(); + var result = new CodexNodeGroup(onlineCodexNodeOrderNumberSource.GetNextNumber(), offline, this, online); onlineCodexNodes.Add(result); return result; } @@ -67,9 +69,9 @@ return containers.ToArray(); } - private OnlineCodexNodes GetAndRemoveActiveNodeFor(IOnlineCodexNodes node) + private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node) { - var n = (OnlineCodexNodes)node; + var n = (CodexNodeGroup)node; onlineCodexNodes.Remove(n); return n; } diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index 38a10d4..55b500d 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -21,7 +21,7 @@ namespace CodexDistTestCore client = new Kubernetes(config); } - public void BringOnline(OnlineCodexNodes online, OfflineCodexNodes offline) + public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline) { EnsureTestNamespace(); @@ -32,7 +32,7 @@ namespace CodexDistTestCore AssignActivePodNames(online); } - public void BringOffline(OnlineCodexNodes online) + public void BringOffline(CodexNodeGroup online) { var deploymentName = online.Deployment.Name(); DeleteDeployment(online); @@ -48,7 +48,7 @@ namespace CodexDistTestCore WaitUntilNamespaceDeleted(); } - public void FetchAllPodsLogs(OnlineCodexNodes[] onlines, IPodLogsHandler logHandler) + public void FetchAllPodsLogs(CodexNodeGroup[] onlines, IPodLogsHandler logHandler) { foreach (var online in onlines) { @@ -61,7 +61,7 @@ namespace CodexDistTestCore } } - private void AssignActivePodNames(OnlineCodexNodes online) + private void AssignActivePodNames(CodexNodeGroup online) { var pods = client.ListNamespacedPod(K8sNamespace); var podNames = pods.Items.Select(p => p.Name()); @@ -77,7 +77,7 @@ namespace CodexDistTestCore #region Waiting - private void WaitUntilOnline(OnlineCodexNodes online) + private void WaitUntilOnline(CodexNodeGroup online) { WaitUntil(() => { @@ -126,7 +126,7 @@ namespace CodexDistTestCore #region Service management - private void CreateService(OnlineCodexNodes online) + private void CreateService(CodexNodeGroup online) { var serviceSpec = new V1Service { @@ -143,7 +143,7 @@ namespace CodexDistTestCore online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); } - private List CreateServicePorts(OnlineCodexNodes online) + private List CreateServicePorts(CodexNodeGroup online) { var result = new List(); var containers = online.GetContainers(); @@ -160,7 +160,7 @@ namespace CodexDistTestCore return result; } - private void DeleteService(OnlineCodexNodes online) + private void DeleteService(CodexNodeGroup online) { if (online.Service == null) return; client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); @@ -171,7 +171,7 @@ namespace CodexDistTestCore #region Deployment management - private void CreateDeployment(OnlineCodexNodes online, OfflineCodexNodes offline) + private void CreateDeployment(CodexNodeGroup online, OfflineCodexNodes offline) { var deploymentSpec = new V1Deployment { @@ -201,7 +201,7 @@ namespace CodexDistTestCore online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); } - private List CreateDeploymentContainers(OnlineCodexNodes online, OfflineCodexNodes offline) + private List CreateDeploymentContainers(CodexNodeGroup online, OfflineCodexNodes offline) { var result = new List(); var containers = online.GetContainers(); @@ -225,7 +225,7 @@ namespace CodexDistTestCore return result; } - private void DeleteDeployment(OnlineCodexNodes online) + private void DeleteDeployment(CodexNodeGroup online) { if (online.Deployment == null) return; client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace); diff --git a/CodexDistTestCore/OfflineCodexNodes.cs b/CodexDistTestCore/OfflineCodexNodes.cs index 10e0093..1b820e2 100644 --- a/CodexDistTestCore/OfflineCodexNodes.cs +++ b/CodexDistTestCore/OfflineCodexNodes.cs @@ -5,7 +5,7 @@ IOfflineCodexNodes WithLogLevel(CodexLogLevel level); IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node); IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota); - IOnlineCodexNodes BringOnline(); + ICodexNodeGroup BringOnline(); } public enum CodexLogLevel @@ -32,7 +32,7 @@ NumberOfNodes = numberOfNodes; } - public IOnlineCodexNodes BringOnline() + public ICodexNodeGroup BringOnline() { return k8SManager.BringOnline(this); } @@ -57,11 +57,15 @@ public string Describe() { - var result = ""; - if (LogLevel != null) result += $"LogLevel={LogLevel},"; - if (BootstrapNode != null) result += "BootstrapNode=set,"; - if (StorageQuota != null) result += $"StorageQuote={StorageQuota.SizeInBytes},"; - return result; + var args = string.Join(',', DescribeArgs()); + return $"{NumberOfNodes} CodexNodes with [{args}]"; + } + + private IEnumerable DescribeArgs() + { + if (LogLevel != null) yield return ($"LogLevel={LogLevel}"); + if (BootstrapNode != null) yield return ("BootstrapNode=set"); + if (StorageQuota != null) yield return ($"StorageQuote={StorageQuota.SizeInBytes}"); } } } diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index 54a1d88..1ce0c11 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -11,38 +11,46 @@ namespace CodexDistTestCore public class OnlineCodexNode : IOnlineCodexNode { + private readonly TestLog log; private readonly IFileManager fileManager; - public OnlineCodexNode(IFileManager fileManager, CodexNodeContainer environment) + public OnlineCodexNode(TestLog log, IFileManager fileManager, CodexNodeContainer container) { + this.log = log; this.fileManager = fileManager; - Container = environment; + Container = container; } public CodexNodeContainer Container { get; } public CodexDebugResponse GetDebugInfo() { - return Http().HttpGetJson("debug/info"); + var response = Http().HttpGetJson("debug/info"); + Log("Got DebugInfo with id: " + response.id); + return response; } public ContentId UploadFile(TestFile file) { + Log($"Uploading file of size {file.GetFileSize()}"); using var fileStream = File.OpenRead(file.Filename); var response = Http().HttpPostStream("upload", fileStream); if (response.StartsWith("Unable to store block")) { Assert.Fail("Node failed to store block."); } + Log($"Uploaded file. Received contentId: {response}"); return new ContentId(response); } public TestFile? DownloadContent(ContentId contentId) { + Log($"Downloading for contentId: {contentId}"); var file = fileManager.CreateEmptyTestFile(); using var fileStream = File.OpenWrite(file.Filename); using var downloadStream = Http().HttpGetStream("download/" + contentId.Id); downloadStream.CopyTo(fileStream); + Log($"Downloaded file of size {file.GetFileSize()}"); return file; } @@ -50,6 +58,11 @@ namespace CodexDistTestCore { return new Http(ip: "127.0.0.1", port: Container.ServicePort, baseUrl: "/api/codex/v1"); } + + private void Log(string msg) + { + log.Log($"Node {Container.Name}: {msg}"); + } } public class ContentId diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index 04cad1c..1ceb6b5 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -5,35 +5,28 @@ namespace CodexDistTestCore public class TestLog { public const string LogRoot = "D:/CodexTestLogs"; + private readonly LogFile file; - private static LogFile? file = null; - - // This is all way too static. It needs to be cleaned up. - public static void Log(string message) + public TestLog() { - file!.Write(message); - } - - public static void Error(string message) - { - Log($"[ERROR] {message}"); - } - - public static void BeginTest() - { - if (file != null) throw new InvalidOperationException("Test is already started!"); - var name = GetTestName(); file = new LogFile(name); Log($"Begin: {name}"); } - public static void EndTest(K8sManager k8sManager) + public void Log(string message) { - if (file == null) throw new InvalidOperationException("No test is started!"); + file.Write(message); + } + public void Error(string message) + { + Log($"[ERROR] {message}"); + } + public void EndTest(K8sManager k8sManager) + { var result = TestContext.CurrentContext.Result; Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); @@ -42,8 +35,6 @@ namespace CodexDistTestCore var logWriter = new PodLogWriter(file); logWriter.IncludeFullPodLogging(k8sManager); } - - file = null; } private static string GetTestName() @@ -65,14 +56,14 @@ namespace CodexDistTestCore public void IncludeFullPodLogging(K8sManager k8sManager) { - TestLog.Log("Full pod logging:"); + file.Write("Full pod logging:"); k8sManager.FetchAllPodsLogs(this); } public void Log(int id, string podDescription, Stream log) { var logFile = id.ToString().PadLeft(6, '0'); - TestLog.Log($"{podDescription} -->> {logFile}"); + file.Write($"{podDescription} -->> {logFile}"); LogRaw(podDescription, logFile); var reader = new StreamReader(log); var line = reader.ReadLine();