wiring up the dist-test backend

CodexDistTestCore/Config/K8sCluster.cs

@@ -4,7 +4,7 @@ namespace CodexDistTestCore.Config
 {
     public class K8sCluster
     {
-        public const string K8sNamespace = "codex-test-namespace";
+        public const string K8sNamespace = "";
         private const string KubeConfigFile = "C:\\kube\\config";
         private readonly Dictionary<Location, string> K8sNodeLocationMap = new Dictionary<Location, string>
         {

DistTestCore/CodexSetupConfig.cs

@@ -0,0 +1,97 @@
+using DistTestCore.Codex;
+
+namespace DistTestCore
+{
+    public interface ICodexSetupConfig
+    {
+        ICodexSetupConfig At(Location location);
+        ICodexSetupConfig WithLogLevel(CodexLogLevel level);
+        //ICodexStartupConfig WithBootstrapNode(IOnlineCodexNode node);
+        ICodexSetupConfig WithStorageQuota(ByteSize storageQuota);
+        ICodexSetupConfig EnableMetrics();
+        //ICodexSetupConfig EnableMarketplace(int initialBalance);
+        ICodexNodeGroup BringOnline();
+    }
+
+    public enum Location
+    {
+        Unspecified,
+        BensLaptop,
+        BensOldGamingMachine,
+    }
+
+    public class CodexSetupConfig : ICodexSetupConfig
+    {
+        private readonly CodexStarter starter;
+
+        public int NumberOfNodes { get; }
+        public Location Location { get; private set; }
+        public CodexLogLevel? LogLevel { get; private set; }
+        //public IOnlineCodexNode? BootstrapNode { get; private set; }
+        public ByteSize? StorageQuota { get; private set; }
+        public bool MetricsEnabled { get; private set; }
+        //public MarketplaceInitialConfig? MarketplaceConfig { get; private set; }
+
+        public CodexSetupConfig(CodexStarter starter, int numberOfNodes)
+        {
+            this.starter = starter;
+            NumberOfNodes = numberOfNodes;
+            Location = Location.Unspecified;
+            MetricsEnabled = false;
+        }
+
+        public ICodexNodeGroup BringOnline()
+        {
+            return starter.BringOnline(this);
+        }
+
+        public ICodexSetupConfig At(Location location)
+        {
+            Location = location;
+            return this;
+        }
+
+        //public ICodexSetupConfig WithBootstrapNode(IOnlineCodexNode node)
+        //{
+        //    BootstrapNode = node;
+        //    return this;
+        //}
+
+        public ICodexSetupConfig WithLogLevel(CodexLogLevel level)
+        {
+            LogLevel = level;
+            return this;
+        }
+
+        public ICodexSetupConfig WithStorageQuota(ByteSize storageQuota)
+        {
+            StorageQuota = storageQuota;
+            return this;
+        }
+
+        public ICodexSetupConfig EnableMetrics()
+        {
+            MetricsEnabled = true;
+            return this;
+        }
+
+        //public ICodexSetupConfig EnableMarketplace(int initialBalance)
+        //{
+        //    MarketplaceConfig = new MarketplaceInitialConfig(initialBalance);
+        //    return this;
+        //}
+
+        public string Describe()
+        {
+            var args = string.Join(',', DescribeArgs());
+            return $"{NumberOfNodes} CodexNodes with [{args}]";
+        }
+
+        private IEnumerable<string> DescribeArgs()
+        {
+            if (LogLevel != null) yield return $"LogLevel={LogLevel}";
+            //if (BootstrapNode != null) yield return "BootstrapNode=set-not-shown-here";
+            if (StorageQuota != null) yield return $"StorageQuote={StorageQuota.SizeInBytes}";
+        }
+    }
+}

DistTestCore/CodexStarter.cs

@@ -0,0 +1,26 @@
+using KubernetesWorkflow;
+using Logging;
+
+namespace DistTestCore
+{
+    public class CodexStarter
+    {
+        private readonly WorkflowCreator workflowCreator;
+
+        public CodexStarter(TestLog log, Configuration configuration)
+        {
+            workflowCreator = new WorkflowCreator(configuration.GetK8sConfiguration());
+        }
+
+        public ICodexNodeGroup BringOnline(CodexSetupConfig codexSetupConfig)
+        {
+            
+        }
+
+        public void DeleteAllResources()
+        {
+            var workflow = workflowCreator.CreateWorkflow();
+            workflow.DeleteAllResources();
+        }
+    }
+}

DistTestCore/Configuration.cs

@@ -0,0 +1,32 @@
+using KubernetesWorkflow;
+
+namespace DistTestCore
+{
+    public class Configuration
+    {
+        public KubernetesWorkflow.Configuration GetK8sConfiguration()
+        {
+            return new KubernetesWorkflow.Configuration(
+                k8sNamespace: "codex-test-ns",
+                kubeConfigFile: null,
+                operationTimeout: Timing.K8sOperationTimeout(),
+                retryDelay: Timing.K8sServiceDelay(),
+                locationMap: new[]
+                {
+                    new ConfigurationLocationEntry(Location.BensOldGamingMachine, "worker01"),
+                    new ConfigurationLocationEntry(Location.BensLaptop, "worker02"),
+                } 
+            );
+        }
+
+        public Logging.LogConfig GetLogConfig()
+        {
+            return new Logging.LogConfig("D:/CodexTestLogs");
+        }
+
+        public string GetFileManagerFolder()
+        {
+            return "TestDataFiles";
+        }
+    }
+}

DistTestCore/DistTest.cs

@@ -0,0 +1,123 @@
+using NUnit.Framework;
+
+namespace DistTestCore
+{
+    [SetUpFixture]
+    public abstract class DistTest
+    {
+        private TestLifecycle lifecycle = null!;
+
+        [OneTimeSetUp]
+        public void GlobalSetup()
+        {
+            // Previous test run may have been interrupted.
+            // Begin by cleaning everything up.
+            CreateNewTestLifecycle();
+
+            try
+            {
+                lifecycle.DeleteAllResources();
+            }
+            catch (Exception ex)
+            {
+                GlobalTestFailure.HasFailed = true;
+                Error($"Global setup cleanup failed with: {ex}");
+                throw;
+            }
+            Log("Global setup cleanup successful");
+        }
+
+        [SetUp]
+        public void SetUpDistTest()
+        {
+            if (GlobalTestFailure.HasFailed)
+            {
+                Assert.Inconclusive("Skip test: Previous test failed during clean up.");
+            }
+            else
+            {
+                CreateNewTestLifecycle();
+            }
+        }
+
+        [TearDown]
+        public void TearDownDistTest()
+        {
+            try
+            {
+                lifecycle.Log.EndTest();
+                IncludeLogsAndMetricsOnTestFailure();
+                lifecycle.DeleteAllResources();
+            }
+            catch (Exception ex)
+            {
+                Error("Cleanup failed: " + ex.Message);
+                GlobalTestFailure.HasFailed = true;
+            }
+        }
+
+        public TestFile GenerateTestFile(ByteSize size)
+        {
+            return lifecycle.FileManager.GenerateTestFile(size);
+        }
+
+        public ICodexSetupConfig SetupCodexNodes(int numberOfNodes)
+        {
+            return new CodexSetupConfig(lifecycle.CodexStarter, numberOfNodes);
+        }
+
+        private void IncludeLogsAndMetricsOnTestFailure()
+        {
+            var result = TestContext.CurrentContext.Result;
+            if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed)
+            {
+                if (IsDownloadingLogsAndMetricsEnabled())
+                {
+                    log.Log("Downloading all CodexNode logs and metrics because of test failure...");
+                    k8sManager.ForEachOnlineGroup(DownloadLogs);
+                    k8sManager.DownloadAllMetrics();
+                }
+                else
+                {
+                    log.Log("Skipping download of all CodexNode logs and metrics due to [DontDownloadLogsAndMetricsOnFailure] attribute.");
+                }
+            }
+        }
+
+        private void Log(string msg)
+        {
+            lifecycle.Log.Log(msg);
+        }
+
+        private void Error(string msg)
+        {
+            lifecycle.Log.Error(msg);
+        }
+
+        private void CreateNewTestLifecycle()
+        {
+            lifecycle = new TestLifecycle(new Configuration());
+        }
+
+        private void DownloadLogs(CodexNodeGroup group)
+        {
+            foreach (var node in group)
+            {
+                var downloader = new PodLogDownloader(log, k8sManager);
+                var n = (OnlineCodexNode)node;
+                downloader.DownloadLog(n);
+            }
+        }
+
+        private bool IsDownloadingLogsAndMetricsEnabled()
+        {
+            var testProperties = TestContext.CurrentContext.Test.Properties;
+            return !testProperties.ContainsKey(PodLogDownloader.DontDownloadLogsOnFailureKey);
+        }
+    }
+
+    public static class GlobalTestFailure
+    {
+        public static bool HasFailed { get; set; } = false;
+    }
+}

DistTestCore/FileManager.cs

@@ -0,0 +1,113 @@
+using Logging;
+using NUnit.Framework;
+
+namespace DistTestCore
+{
+    public interface IFileManager
+    {
+        TestFile CreateEmptyTestFile();
+        TestFile GenerateTestFile(ByteSize size);
+        void DeleteAllTestFiles();
+    }
+
+    public class FileManager : IFileManager
+    {
+        public const int ChunkSize = 1024 * 1024;
+        private readonly Random random = new Random();
+        private readonly List<TestFile> activeFiles = new List<TestFile>();
+        private readonly TestLog log;
+        private readonly string folder;
+
+        public FileManager(TestLog log, Configuration configuration)
+        {
+            folder = configuration.GetFileManagerFolder();
+
+            if (!Directory.Exists(folder)) Directory.CreateDirectory(folder);
+            this.log = log;
+        }
+
+        public TestFile CreateEmptyTestFile()
+        {
+            var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin"));
+            File.Create(result.Filename).Close();
+            activeFiles.Add(result);
+            return result;
+        }
+
+        public TestFile GenerateTestFile(ByteSize size)
+        {
+            var result = CreateEmptyTestFile();
+            GenerateFileBytes(result, size);
+            log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'.");
+            return result;
+        }
+
+        public void DeleteAllTestFiles()
+        {
+            foreach (var file in activeFiles) File.Delete(file.Filename);
+            activeFiles.Clear();
+        }
+
+        private void GenerateFileBytes(TestFile result, ByteSize size)
+        {
+            long bytesLeft = size.SizeInBytes;
+            while (bytesLeft > 0)
+            {
+                var length = Math.Min(bytesLeft, ChunkSize);
+                AppendRandomBytesToFile(result, length);
+                bytesLeft -= length;
+            }
+        }
+
+        private void AppendRandomBytesToFile(TestFile result, long length)
+        {
+            var bytes = new byte[length];
+            random.NextBytes(bytes);
+            using var stream = new FileStream(result.Filename, FileMode.Append);
+            stream.Write(bytes, 0, bytes.Length);
+        }
+    }
+
+    public class TestFile
+    {
+        public TestFile(string filename)
+        {
+            Filename = filename;
+        }
+
+        public string Filename { get; }
+
+        public long GetFileSize()
+        {
+            var info = new FileInfo(Filename);
+            return info.Length;
+        }
+
+        public void AssertIsEqual(TestFile? actual)
+        {
+            if (actual == null) Assert.Fail("TestFile is null.");
+            if (actual == this || actual!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
+
+            Assert.That(actual.GetFileSize(), Is.EqualTo(GetFileSize()), "Files are not of equal length.");
+
+            using var streamExpected = new FileStream(Filename, FileMode.Open, FileAccess.Read);
+            using var streamActual = new FileStream(actual.Filename, FileMode.Open, FileAccess.Read);
+
+            var bytesExpected = new byte[FileManager.ChunkSize];
+            var bytesActual = new byte[FileManager.ChunkSize];
+
+            var readExpected = 0;
+            var readActual = 0;
+
+            while (true)
+            {
+                readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize);
+                readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize);
+
+                if (readExpected == 0 && readActual == 0) return;
+                Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length.");
+                CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal.");
+            }
+        }
+    }
+}

DistTestCore/TestLifecycle.cs

@@ -1,31 +1,24 @@
-using DistTestCore.Codex;
-using KubernetesWorkflow;
+using Logging;
 
 namespace DistTestCore
 {
     public class TestLifecycle
     {
-        private readonly WorkflowCreator workflowCreator = new WorkflowCreator();
-
-        public void SetUpCodexNodes()
+        public TestLifecycle(Configuration configuration)
         {
-            var config = new CodexStartupConfig()
-            {
-                StorageQuota = 10.MB(),
-                Location = Location.Unspecified,
-                LogLevel = CodexLogLevel.Error,
-                MetricsEnabled = false,
-            };
+            Log = new TestLog(configuration.GetLogConfig());
+            FileManager = new FileManager(Log, configuration);
+            CodexStarter = new CodexStarter(Log, configuration);
+        }
 
-            var workflow = workflowCreator.CreateWorkflow();
-            var startupConfig = new StartupConfig();
-            startupConfig.Add(config);
-            var containers = workflow.Start(3, new CodexContainerRecipe(), startupConfig);
+        public TestLog Log { get; }
+        public FileManager FileManager { get; }
+        public CodexStarter CodexStarter { get; }
 
-            foreach (var c in containers.Containers)
-            {
-                var access = new CodexAccess(c);
-            }
+        public void DeleteAllResources()
+        {
+            CodexStarter.DeleteAllResources();
+            FileManager.DeleteAllTestFiles();
         }
     }
 }

DistTestCore/Timing.cs

@@ -0,0 +1,132 @@
+using NUnit.Framework;
+using Utils;
+
+namespace DistTestCore
+{
+    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)]
+    public class UseLongTimeoutsAttribute : PropertyAttribute
+    {
+        public UseLongTimeoutsAttribute()
+            : base(Timing.UseLongTimeoutsKey)
+        {
+        }
+    }
+
+    public static class Timing
+    {
+        public const string UseLongTimeoutsKey = "UseLongTimeouts";
+
+        public static TimeSpan HttpCallTimeout()
+        {
+            return GetTimes().HttpCallTimeout();
+        }
+
+        public static int HttpCallRetryCount()
+        {
+            return GetTimes().HttpCallRetryCount();
+        }
+
+        public static void HttpCallRetryDelay()
+        {
+            Time.Sleep(GetTimes().HttpCallRetryDelay());
+        }
+
+        public static TimeSpan K8sServiceDelay()
+        {
+            return GetTimes().WaitForK8sServiceDelay();
+        }
+
+        public static TimeSpan K8sOperationTimeout()
+        {
+            return GetTimes().K8sOperationTimeout();
+        }
+
+        public static TimeSpan WaitForMetricTimeout()
+        {
+            return GetTimes().WaitForMetricTimeout();
+        }
+
+        private static ITimeSet GetTimes()
+        {
+            var testProperties = TestContext.CurrentContext.Test.Properties;
+            if (testProperties.ContainsKey(UseLongTimeoutsKey)) return new LongTimeSet();
+            return new DefaultTimeSet();
+        }
+    }
+
+    public interface ITimeSet
+    {
+        TimeSpan HttpCallTimeout();
+        int HttpCallRetryCount();
+        TimeSpan HttpCallRetryDelay();
+        TimeSpan WaitForK8sServiceDelay();
+        TimeSpan K8sOperationTimeout();
+        TimeSpan WaitForMetricTimeout();
+    }
+
+    public class DefaultTimeSet : ITimeSet
+    {
+        public TimeSpan HttpCallTimeout()
+        {
+            return TimeSpan.FromSeconds(10);
+        }
+
+        public int HttpCallRetryCount()
+        {
+            return 5;
+        }
+
+        public TimeSpan HttpCallRetryDelay()
+        {
+            return TimeSpan.FromSeconds(3);
+        }
+
+        public TimeSpan WaitForK8sServiceDelay()
+        {
+            return TimeSpan.FromSeconds(1);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromMinutes(5);
+        }
+
+        public TimeSpan WaitForMetricTimeout()
+        {
+            return TimeSpan.FromSeconds(30);
+        }
+    }
+
+    public class LongTimeSet : ITimeSet
+    {
+        public TimeSpan HttpCallTimeout()
+        {
+            return TimeSpan.FromHours(2);
+        }
+
+        public int HttpCallRetryCount()
+        {
+            return 2;
+        }
+
+        public TimeSpan HttpCallRetryDelay()
+        {
+            return TimeSpan.FromMinutes(5);
+        }
+
+        public TimeSpan WaitForK8sServiceDelay()
+        {
+            return TimeSpan.FromSeconds(10);
+        }
+
+        public TimeSpan K8sOperationTimeout()
+        {
+            return TimeSpan.FromMinutes(15);
+        }
+
+        public TimeSpan WaitForMetricTimeout()
+        {
+            return TimeSpan.FromMinutes(5);
+        }
+    }
+}

Logging/LogConfig.cs

@@ -2,6 +2,11 @@
 {
     public class LogConfig
     {
-        public const string LogRoot = "D:/CodexTestLogs";
+        public LogConfig(string logRoot)
+        {
+            LogRoot = logRoot;
+        }
+        
+        public string LogRoot { get; }
     }
 }

Logging/LogFile.cs

@@ -7,14 +7,14 @@
         private readonly string ext;
         private readonly string filepath;
 
-        public LogFile(DateTime now, string name, string ext = "log")
+        public LogFile(LogConfig config, DateTime now, string name, string ext = "log")
         {
             this.now = now;
             this.name = name;
             this.ext = ext;
 
             filepath = Path.Join(
-                LogConfig.LogRoot,
+                config.LogRoot,
                 $"{now.Year}-{Pad(now.Month)}",
                 Pad(now.Day));
 

Logging/TestLog.cs

@@ -8,13 +8,15 @@ namespace Logging
         private readonly NumberSource subfileNumberSource = new NumberSource(0);
         private readonly LogFile file;
         private readonly DateTime now;
+        private readonly LogConfig config;
 
-        public TestLog()
+        public TestLog(LogConfig config)
         {
+            this.config = config;
             now = DateTime.UtcNow;
 
             var name = GetTestName();
-            file = new LogFile(now, name);
+            file = new LogFile(config, now, name);
 
             Log($"Begin: {name}");
         }
@@ -53,7 +55,7 @@ namespace Logging
 
         public LogFile CreateSubfile(string ext = "log")
         {
-            return new LogFile(now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext);
+            return new LogFile(config, now, $"{GetTestName()}_{subfileNumberSource.GetNextNumber().ToString().PadLeft(6, '0')}", ext);
         }
 
         private static string GetTestName()
@@ -70,5 +72,4 @@ namespace Logging
             return $"[{string.Join(',', test.Arguments)}]";
         }
     }
-
 }