diff --git a/CodexDistTestCore/Config/K8sCluster.cs b/CodexDistTestCore/Config/K8sCluster.cs index a290193f..a17e6fdb 100644 --- a/CodexDistTestCore/Config/K8sCluster.cs +++ b/CodexDistTestCore/Config/K8sCluster.cs @@ -4,7 +4,7 @@ namespace CodexDistTestCore.Config { public class K8sCluster { - public const string K8sNamespace = "codex-test-namespace"; + public const string K8sNamespace = ""; private const string KubeConfigFile = "C:\\kube\\config"; private readonly Dictionary K8sNodeLocationMap = new Dictionary { diff --git a/DistTestCore/CodexSetupConfig.cs b/DistTestCore/CodexSetupConfig.cs new file mode 100644 index 00000000..1f1a6328 --- /dev/null +++ b/DistTestCore/CodexSetupConfig.cs @@ -0,0 +1,97 @@ +using DistTestCore.Codex; + +namespace DistTestCore +{ + public interface ICodexSetupConfig + { + ICodexSetupConfig At(Location location); + ICodexSetupConfig WithLogLevel(CodexLogLevel level); + //ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node); + ICodexSetupConfig WithStorageQuota(ByteSize storageQuota); + ICodexSetupConfig EnableMetrics(); + //ICodexSetupConfig EnableMarketplace(int initialBalance); + ICodexNodeGroup BringOnline(); + } + + public enum Location + { + Unspecified, + BensLaptop, + BensOldGamingMachine, + } + + public class CodexSetupConfig : ICodexSetupConfig + { + private readonly CodexStarter starter; + + public int NumberOfNodes { get; } + public Location Location { get; private set; } + public CodexLogLevel? LogLevel { get; private set; } + //public IOnlineCodexNode? BootstrapNode { get; private set; } + public ByteSize? StorageQuota { get; private set; } + public bool MetricsEnabled { get; private set; } + //public MarketplaceInitialConfig? MarketplaceConfig { get; private set; } + + public CodexSetupConfig(CodexStarter starter, int numberOfNodes) + { + this.starter = starter; + NumberOfNodes = numberOfNodes; + Location = Location.Unspecified; + MetricsEnabled = false; + } + + public ICodexNodeGroup BringOnline() + { + return starter.BringOnline(this); + } + + public ICodexSetupConfig At(Location location) + { + Location = location; + return this; + } + + //public ICodexSetupConfig WithBootstrapNode(IOnlineCodexNode node) + //{ + // BootstrapNode = node; + // return this; + //} + + public ICodexSetupConfig WithLogLevel(CodexLogLevel level) + { + LogLevel = level; + return this; + } + + public ICodexSetupConfig WithStorageQuota(ByteSize storageQuota) + { + StorageQuota = storageQuota; + return this; + } + + public ICodexSetupConfig EnableMetrics() + { + MetricsEnabled = true; + return this; + } + + //public ICodexSetupConfig EnableMarketplace(int initialBalance) + //{ + // MarketplaceConfig = new MarketplaceInitialConfig(initialBalance); + // return this; + //} + + public string Describe() + { + var args = string.Join(',', DescribeArgs()); + return $"{NumberOfNodes} CodexNodes with [{args}]"; + } + + private IEnumerable DescribeArgs() + { + if (LogLevel != null) yield return $"LogLevel={LogLevel}"; + //if (BootstrapNode != null) yield return "BootstrapNode=set-not-shown-here"; + if (StorageQuota != null) yield return $"StorageQuote={StorageQuota.SizeInBytes}"; + } + } +} diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs new file mode 100644 index 00000000..f295d0a9 --- /dev/null +++ b/DistTestCore/CodexStarter.cs @@ -0,0 +1,26 @@ +using KubernetesWorkflow; +using Logging; + +namespace DistTestCore +{ + public class CodexStarter + { + private readonly WorkflowCreator workflowCreator; + + public CodexStarter(TestLog log, Configuration configuration) + { + workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration()); + } + + public ICodexNodeGroup BringOnline(CodexSetupConfig codexSetupConfig) + { + + } + + public void DeleteAllResources() + { + var workflow = workflowCreator.CreateWorkflow(); + workflow.DeleteAllResources(); + } + } +} diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs new file mode 100644 index 00000000..f11437c5 --- /dev/null +++ b/DistTestCore/Configuration.cs @@ -0,0 +1,32 @@ +using KubernetesWorkflow; + +namespace DistTestCore +{ + public class Configuration + { + public KubernetesWorkflow.Configuration GetK8sConfiguration() + { + return new KubernetesWorkflow.Configuration( + k8sNamespace: "codex-test-ns", + kubeConfigFile: null, + operationTimeout: Timing.K8sOperationTimeout(), + retryDelay: Timing.K8sServiceDelay(), + locationMap: new[] + { + new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"), + new ConfigurationLocationEntry(Location.BensLaptop, "worker02"), + } + ); + } + + public Logging.LogConfig GetLogConfig() + { + return new Logging.LogConfig("D:/CodexTestLogs"); + } + + public string GetFileManagerFolder() + { + return "TestDataFiles"; + } + } +} diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs new file mode 100644 index 00000000..995fffa4 --- /dev/null +++ b/DistTestCore/DistTest.cs @@ -0,0 +1,123 @@ +using NUnit.Framework; + +namespace DistTestCore +{ + [SetUpFixture] + public abstract class DistTest + { + private TestLifecycle lifecycle = null!; + + [OneTimeSetUp] + public void GlobalSetup() + { + // Previous test run may have been interrupted. + // Begin by cleaning everything up. + CreateNewTestLifecycle(); + + try + { + lifecycle.DeleteAllResources(); + } + catch (Exception ex) + { + GlobalTestFailure.HasFailed = true; + Error($"Global setup cleanup failed with: {ex}"); + throw; + } + Log("Global setup cleanup successful"); + } + + [SetUp] + public void SetUpDistTest() + { + if (GlobalTestFailure.HasFailed) + { + Assert.Inconclusive("Skip test: Previous test failed during clean up."); + } + else + { + CreateNewTestLifecycle(); + } + } + + [TearDown] + public void TearDownDistTest() + { + try + { + lifecycle.Log.EndTest(); + IncludeLogsAndMetricsOnTestFailure(); + lifecycle.DeleteAllResources(); + } + catch (Exception ex) + { + Error("Cleanup failed: " + ex.Message); + GlobalTestFailure.HasFailed = true; + } + } + + public TestFile GenerateTestFile(ByteSize size) + { + return lifecycle.FileManager.GenerateTestFile(size); + } + + public ICodexSetupConfig SetupCodexNodes(int numberOfNodes) + { + return new CodexSetupConfig(lifecycle.CodexStarter, numberOfNodes); + } + + private void IncludeLogsAndMetricsOnTestFailure() + { + var result = TestContext.CurrentContext.Result; + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + if (IsDownloadingLogsAndMetricsEnabled()) + { + log.Log("Downloading all CodexNode logs and metrics because of test failure..."); + k8sManager.ForEachOnlineGroup(DownloadLogs); + k8sManager.DownloadAllMetrics(); + } + else + { + log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute."); + } + } + } + + private void Log(string msg) + { + lifecycle.Log.Log(msg); + } + + private void Error(string msg) + { + lifecycle.Log.Error(msg); + } + + private void CreateNewTestLifecycle() + { + lifecycle = new TestLifecycle(new Configuration()); + } + + private void DownloadLogs(CodexNodeGroup group) + { + foreach (var node in group) + { + var downloader = new PodLogDownloader(log, k8sManager); + var n = (OnlineCodexNode)node; + downloader.DownloadLog(n); + } + } + + private bool IsDownloadingLogsAndMetricsEnabled() + { + var testProperties = TestContext.CurrentContext.Test.Properties; + return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey); + } + } + + public static class GlobalTestFailure + { + public static bool HasFailed { get; set; } = false; + } +} diff --git a/DistTestCore/FileManager.cs b/DistTestCore/FileManager.cs new file mode 100644 index 00000000..10f126b5 --- /dev/null +++ b/DistTestCore/FileManager.cs @@ -0,0 +1,113 @@ +using Logging; +using NUnit.Framework; + +namespace DistTestCore +{ + public interface IFileManager + { + TestFile CreateEmptyTestFile(); + TestFile GenerateTestFile(ByteSize size); + void DeleteAllTestFiles(); + } + + public class FileManager : IFileManager + { + public const int ChunkSize = 1024 * 1024; + private readonly Random random = new Random(); + private readonly List activeFiles = new List(); + private readonly TestLog log; + private readonly string folder; + + public FileManager(TestLog log, Configuration configuration) + { + folder = configuration.GetFileManagerFolder(); + + if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); + this.log = log; + } + + public TestFile CreateEmptyTestFile() + { + var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin")); + File.Create(result.Filename).Close(); + activeFiles.Add(result); + return result; + } + + public TestFile GenerateTestFile(ByteSize size) + { + var result = CreateEmptyTestFile(); + GenerateFileBytes(result, size); + log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'."); + return result; + } + + public void DeleteAllTestFiles() + { + foreach (var file in activeFiles) File.Delete(file.Filename); + activeFiles.Clear(); + } + + private void GenerateFileBytes(TestFile result, ByteSize size) + { + long bytesLeft = size.SizeInBytes; + while (bytesLeft > 0) + { + var length = Math.Min(bytesLeft, ChunkSize); + AppendRandomBytesToFile(result, length); + bytesLeft -= length; + } + } + + private void AppendRandomBytesToFile(TestFile result, long length) + { + var bytes = new byte[length]; + random.NextBytes(bytes); + using var stream = new FileStream(result.Filename, FileMode.Append); + stream.Write(bytes, 0, bytes.Length); + } + } + + public class TestFile + { + public TestFile(string filename) + { + Filename = filename; + } + + public string Filename { get; } + + public long GetFileSize() + { + var info = new FileInfo(Filename); + return info.Length; + } + + public void AssertIsEqual(TestFile? actual) + { + if (actual == null) Assert.Fail("TestFile is null."); + if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); + + Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length."); + + using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read); + using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read); + + var bytesExpected = new byte[FileManager.ChunkSize]; + var bytesActual = new byte[FileManager.ChunkSize]; + + var readExpected = 0; + var readActual = 0; + + while (true) + { + readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize); + readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize); + + if (readExpected == 0 && readActual == 0) return; + Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length."); + CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal."); + } + } + } +} diff --git a/DistTestCore/TestLifecycle.cs b/DistTestCore/TestLifecycle.cs index e3d21867..6ff92f97 100644 --- a/DistTestCore/TestLifecycle.cs +++ b/DistTestCore/TestLifecycle.cs @@ -1,31 +1,24 @@ -using DistTestCore.Codex; -using KubernetesWorkflow; +using Logging; namespace DistTestCore { public class TestLifecycle { - private readonly WorkflowCreator workflowCreator = new WorkflowCreator(); - - public void SetUpCodexNodes() + public TestLifecycle(Configuration configuration) { - var config = new CodexStartupConfig() - { - StorageQuota = 10.MB(), - Location = Location.Unspecified, - LogLevel = CodexLogLevel.Error, - MetricsEnabled = false, - }; + Log = new TestLog(configuration.GetLogConfig()); + FileManager = new FileManager(Log, configuration); + CodexStarter = new CodexStarter(Log, configuration); + } - var workflow = workflowCreator.CreateWorkflow(); - var startupConfig = new StartupConfig(); - startupConfig.Add(config); - var containers = workflow.Start(3, new CodexContainerRecipe(), startupConfig); + public TestLog Log { get; } + public FileManager FileManager { get; } + public CodexStarter CodexStarter { get; } - foreach (var c in containers.Containers) - { - var access = new CodexAccess(c); - } + public void DeleteAllResources() + { + CodexStarter.DeleteAllResources(); + FileManager.DeleteAllTestFiles(); } } } diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs new file mode 100644 index 00000000..67653ece --- /dev/null +++ b/DistTestCore/Timing.cs @@ -0,0 +1,132 @@ +using NUnit.Framework; +using Utils; + +namespace DistTestCore +{ + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public class UseLongTimeoutsAttribute : PropertyAttribute + { + public UseLongTimeoutsAttribute() + : base(Timing.UseLongTimeoutsKey) + { + } + } + + public static class Timing + { + public const string UseLongTimeoutsKey = "UseLongTimeouts"; + + public static TimeSpan HttpCallTimeout() + { + return GetTimes().HttpCallTimeout(); + } + + public static int HttpCallRetryCount() + { + return GetTimes().HttpCallRetryCount(); + } + + public static void HttpCallRetryDelay() + { + Time.Sleep(GetTimes().HttpCallRetryDelay()); + } + + public static TimeSpan K8sServiceDelay() + { + return GetTimes().WaitForK8sServiceDelay(); + } + + public static TimeSpan K8sOperationTimeout() + { + return GetTimes().K8sOperationTimeout(); + } + + public static TimeSpan WaitForMetricTimeout() + { + return GetTimes().WaitForMetricTimeout(); + } + + private static ITimeSet GetTimes() + { + var testProperties = TestContext.CurrentContext.Test.Properties; + if (testProperties.ContainsKey(UseLongTimeoutsKey)) return new LongTimeSet(); + return new DefaultTimeSet(); + } + } + + public interface ITimeSet + { + TimeSpan HttpCallTimeout(); + int HttpCallRetryCount(); + TimeSpan HttpCallRetryDelay(); + TimeSpan WaitForK8sServiceDelay(); + TimeSpan K8sOperationTimeout(); + TimeSpan WaitForMetricTimeout(); + } + + public class DefaultTimeSet : ITimeSet + { + public TimeSpan HttpCallTimeout() + { + return TimeSpan.FromSeconds(10); + } + + public int HttpCallRetryCount() + { + return 5; + } + + public TimeSpan HttpCallRetryDelay() + { + return TimeSpan.FromSeconds(3); + } + + public TimeSpan WaitForK8sServiceDelay() + { + return TimeSpan.FromSeconds(1); + } + + public TimeSpan K8sOperationTimeout() + { + return TimeSpan.FromMinutes(5); + } + + public TimeSpan WaitForMetricTimeout() + { + return TimeSpan.FromSeconds(30); + } + } + + public class LongTimeSet : ITimeSet + { + public TimeSpan HttpCallTimeout() + { + return TimeSpan.FromHours(2); + } + + public int HttpCallRetryCount() + { + return 2; + } + + public TimeSpan HttpCallRetryDelay() + { + return TimeSpan.FromMinutes(5); + } + + public TimeSpan WaitForK8sServiceDelay() + { + return TimeSpan.FromSeconds(10); + } + + public TimeSpan K8sOperationTimeout() + { + return TimeSpan.FromMinutes(15); + } + + public TimeSpan WaitForMetricTimeout() + { + return TimeSpan.FromMinutes(5); + } + } +} diff --git a/Logging/LogConfig.cs b/Logging/LogConfig.cs index c15b1fe9..b7bc937c 100644 --- a/Logging/LogConfig.cs +++ b/Logging/LogConfig.cs @@ -2,6 +2,11 @@ { public class LogConfig { - public const string LogRoot = "D:/CodexTestLogs"; + public LogConfig(string logRoot) + { + LogRoot = logRoot; + } + + public string LogRoot { get; } } } diff --git a/Logging/LogFile.cs b/Logging/LogFile.cs index a43186c5..3a0063ba 100644 --- a/Logging/LogFile.cs +++ b/Logging/LogFile.cs @@ -7,14 +7,14 @@ private readonly string ext; private readonly string filepath; - public LogFile(DateTime now, string name, string ext = "log") + public LogFile(LogConfig config, DateTime now, string name, string ext = "log") { this.now = now; this.name = name; this.ext = ext; filepath = Path.Join( - LogConfig.LogRoot, + config.LogRoot, $"{now.Year}-{Pad(now.Month)}", Pad(now.Day)); diff --git a/Logging/TestLog.cs b/Logging/TestLog.cs index 83b6cb18..7eb3979d 100644 --- a/Logging/TestLog.cs +++ b/Logging/TestLog.cs @@ -8,13 +8,15 @@ namespace Logging private readonly NumberSource subfileNumberSource = new NumberSource(0); private readonly LogFile file; private readonly DateTime now; + private readonly LogConfig config; - public TestLog() + public TestLog(LogConfig config) { + this.config = config; now = DateTime.UtcNow; var name = GetTestName(); - file = new LogFile(now, name); + file = new LogFile(config, now, name); Log($"Begin: {name}"); } @@ -53,7 +55,7 @@ namespace Logging public LogFile CreateSubfile(string ext = "log") { - return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); + return new LogFile(config, now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext); } private static string GetTestName() @@ -70,5 +72,4 @@ namespace Logging return $"[{string.Join(',', test.Arguments)}]"; } } - }