using CodexDistTestCore.Config; using NUnit.Framework; namespace CodexDistTestCore { [SetUpFixture] public abstract class DistTest { private TestLog log = null!; private FileManager fileManager = null!; private K8sManager k8sManager = null!; private MetricsAggregator metricsAggregator = null!; [OneTimeSetUp] public void GlobalSetup() { // Previous test run may have been interrupted. // Begin by cleaning everything up. log = new TestLog(); fileManager = new FileManager(log); k8sManager = new K8sManager(log, fileManager); try { k8sManager.DeleteAllResources(); fileManager.DeleteAllTestFiles(); } catch (Exception ex) { GlobalTestFailure.HasFailed = true; log.Error($"Global setup cleanup failed with: {ex}"); throw; } log.Log("Global setup cleanup successful"); } [SetUp] public void SetUpDistTest() { if (GlobalTestFailure.HasFailed) { Assert.Inconclusive("Skip test: Previous test failed during clean up."); } else { var dockerImage = new CodexDockerImage(); log = new TestLog(); log.Log($"Using docker image '{dockerImage.GetImageTag()}'"); fileManager = new FileManager(log); k8sManager = new K8sManager(log, fileManager); metricsAggregator = new MetricsAggregator(log, k8sManager); } } [TearDown] public void TearDownDistTest() { try { log.EndTest(); IncludeLogsAndMetricsOnTestFailure(); k8sManager.DeleteAllResources(); fileManager.DeleteAllTestFiles(); } catch (Exception ex) { log.Error("Cleanup failed: " + ex.Message); GlobalTestFailure.HasFailed = true; } } public TestFile GenerateTestFile(ByteSize size) { return fileManager.GenerateTestFile(size); } public IOfflineCodexNodes SetupCodexNodes(int numberOfNodes) { return new OfflineCodexNodes(k8sManager, numberOfNodes); } public MetricsAccess GatherMetrics(ICodexNodeGroup group) { return GatherMetrics(group.ToArray()); } public MetricsAccess GatherMetrics(params IOnlineCodexNode[] nodes) { var onlineNodes = nodes.Cast().ToArray(); Assert.That(onlineNodes.All(n => n.Group.Origin.MetricsEnabled), "Incorrect test setup: Metrics were not enabled on (all) provided OnlineCodexNodes. " + "To use metrics, please use 'EnableMetrics()' when setting up Codex nodes."); return metricsAggregator.BeginCollectingMetricsFor(onlineNodes); } private void IncludeLogsAndMetricsOnTestFailure() { var result = TestContext.CurrentContext.Result; if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) { if (IsDownloadingLogsAndMetricsEnabled()) { log.Log("Downloading all CodexNode logs and metrics because of test failure..."); k8sManager.ForEachOnlineGroup(DownloadLogs); metricsAggregator.DownloadAllMetrics(); } else { log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); } } } private void DownloadLogs(CodexNodeGroup group) { foreach (var node in group) { var downloader = new PodLogDownloader(log, k8sManager); var n = (OnlineCodexNode)node; downloader.DownloadLog(n); } } private bool IsDownloadingLogsAndMetricsEnabled() { var testProperties = TestContext.CurrentContext.Test.Properties; return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); } } public static class GlobalTestFailure { public static bool HasFailed { get; set; } = false; } }