Cleanup and split up for test supporting methods

BasicTests/DebugEndpointTests.cs

@@ -9,14 +9,35 @@ namespace CodexDistTests.BasicTests
         [Test]
         public void GetDebugInfo()
         {
-            CreateCodexNode();
+            var node = SetupCodexNode().BringOnline();
 
-            var node = GetCodexNode();
             var debugInfo = node.GetDebugInfo();
 
             Assert.That(debugInfo.spr, Is.Not.Empty);
-
-            DestroyCodexNode();
         }
+
+        //[Test]
+        //public void TwoClientTest()
+        //{
+        //    var primaryNodex = SetupCodexNode()
+        //                        .WithLogLevel(CodexLogLevel.Warn)
+        //                        .WithStorageQuota(1024 * 1024)
+        //                        .BringOnline();
+
+        //    var secondaryNodex = SetupCodexNode()
+        //                        .WithBootstrapNode(primaryNodex)
+        //                        .BringOnline();
+
+        //    var testFile = GenerateTestFile(1024 * 1024);
+
+        //    var contentId = primaryNodex.UploadFile(testFile);
+
+        //    var downloadedFile = secondaryNodex.DownloadContent(contentId);
+
+        //    testFile.AssertIsEqual(downloadedFile);
+
+        //    // Test files are automatically deleted.
+        //    // Online nodes are automatically destroyed.
+        //}
     }
 }

TestCore/DistTest.cs

@@ -1,139 +1,34 @@
-using k8s;
+using NUnit.Framework;
-using k8s.Models;
 
 namespace CodexDistTests.TestCore
 {
     public abstract class DistTest
     {
-        private const string k8sNamespace = "codex-test-namespace";
+        private FileManager fileManager = null!;
+        private K8sManager k8sManager = null!;
 
-        private V1Namespace? activeNamespace;
+        [SetUp]
-        private V1Deployment? activeDeployment;
+        public void SetUpDistTest()
-        private V1Service? activeService;
-
-        public void CreateCodexNode()
         {
-            var config = KubernetesClientConfiguration.BuildConfigFromConfigFile();
+            fileManager = new FileManager();
-            var client = new Kubernetes(config);
+            k8sManager = new K8sManager(fileManager);
-
-            var namespaceSpec = new V1Namespace
-            {
-                ApiVersion = "v1",
-                Metadata = new V1ObjectMeta
-                {
-                    Name = k8sNamespace,
-                    Labels = new Dictionary<string, string> { { "name", k8sNamespace } }
-                }
-            };
-            var deploymentSpec = new V1Deployment
-            {
-                ApiVersion = "apps/v1",
-                Metadata = new V1ObjectMeta
-                {
-                    Name = "codex-demo",
-                    NamespaceProperty = k8sNamespace
-                },
-                Spec = new V1DeploymentSpec
-                {
-                    Replicas = 1,
-                    Selector = new V1LabelSelector
-                    {
-                        MatchLabels = new Dictionary<string, string> { { "codex-node", "dist-test" } }
-                    },
-                    Template = new V1PodTemplateSpec
-                    {
-                        Metadata = new V1ObjectMeta
-                        {
-                            Labels = new Dictionary<string, string> { { "codex-node", "dist-test" } }
-                        },
-                        Spec = new V1PodSpec
-                        {
-                            Containers = new List<V1Container>
-                        {
-                            new V1Container
-                            {
-                                Name = "codex-node",
-                                Image = "thatbenbierens/nim-codex:sha-c9a62de",
-                                Ports = new List<V1ContainerPort>
-                                {
-                                    new V1ContainerPort
-                                    {
-                                        ContainerPort = 8080,
-                                        Name = "codex-api-port"
-                                    }
-                                },
-                                Env = new List<V1EnvVar>
-                                {
-                                    new V1EnvVar
-                                    {
-                                        Name = "LOG_LEVEL",
-                                        Value = "WARN"
-                                    }
-                                }
-                            }
-                        }
-                        }
-                    }
-                }
-            };
-            var serviceSpec = new V1Service
-            {
-                ApiVersion = "v1",
-                Metadata = new V1ObjectMeta
-                {
-                    Name = "codex-entrypoint",
-                    NamespaceProperty = k8sNamespace
-                },
-                Spec = new V1ServiceSpec
-                {
-                    Type = "NodePort",
-                    Selector = new Dictionary<string, string> { { "codex-node", "dist-test" } },
-                    Ports = new List<V1ServicePort>
-                {
-                    new V1ServicePort
-                    {
-                        Protocol = "TCP",
-                        Port = 8080,
-                        TargetPort = "codex-api-port",
-                        NodePort = 30001
-                    }
-                }
-                }
-            };
-
-            activeNamespace = client.CreateNamespace(namespaceSpec);
-            activeDeployment = client.CreateNamespacedDeployment(deploymentSpec, k8sNamespace);
-            activeService = client.CreateNamespacedService(serviceSpec, k8sNamespace);
-
-            // todo: wait until online!
-            while (activeDeployment.Status.AvailableReplicas == null || activeDeployment.Status.AvailableReplicas != 1)
-            {
-                Timing.WaitForServiceDelay();
-                activeDeployment = client.ReadNamespacedDeployment(activeDeployment.Name(), k8sNamespace);
-            }
         }
 
-        public CodexNode GetCodexNode()
+        [TearDown]
+        public void TearDownDistTest()
         {
-            return new CodexNode(30001); // matches service spec.
+            fileManager.DeleteAllTestFiles();
+            k8sManager.DeleteAllResources();
         }
 
-        public void DestroyCodexNode()
+        public TestFile GenerateTestFile(int size = 1024)
         {
-            var config = KubernetesClientConfiguration.BuildConfigFromConfigFile();
+            return fileManager.GenerateTestFile(size);
-            var client = new Kubernetes(config);
-
-            client.DeleteNamespacedService(activeService.Name(), k8sNamespace);
-            client.DeleteNamespacedDeployment(activeDeployment.Name(), k8sNamespace);
-            client.DeleteNamespace(activeNamespace.Name());
-
-            // todo: wait until terminated!
-            var pods = client.ListNamespacedPod(k8sNamespace);
-            while (pods.Items.Any())
-            {
-                Timing.WaitForServiceDelay();
-                pods = client.ListNamespacedPod(k8sNamespace);
         }
+
+        public IOfflineCodexNode SetupCodexNode()
+        {
+            return new OfflineCodexNode(k8sManager);
         }
     }
 }

TestCore/FileManager.cs

@@ -0,0 +1,92 @@
+using NUnit.Framework;
+
+namespace CodexDistTests.TestCore
+{
+    public interface IFileManager
+    {
+        TestFile CreateEmptyTestFile();
+        TestFile GenerateTestFile(int size = 1024);
+        void DeleteAllTestFiles();
+    }
+
+    public class FileManager : IFileManager
+    {
+        public const int ChunkSize = 1024;
+        private const string Folder = "TestDataFiles";
+        private readonly Random random = new Random();
+        private readonly List<TestFile> activeFiles = new List<TestFile>();
+
+        public TestFile CreateEmptyTestFile()
+        {
+            var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin"));
+            activeFiles.Add(result);
+            return result;
+        }
+
+        public TestFile GenerateTestFile(int size = 1024)
+        {
+            var result = CreateEmptyTestFile();
+            GenerateFileBytes(result, size);
+            return result;
+        }
+
+        public void DeleteAllTestFiles()
+        {
+            foreach (var file in activeFiles) File.Delete(file.Filename);
+            activeFiles.Clear();
+        }
+
+        private void GenerateFileBytes(TestFile result, int size)
+        {
+            while (size > 0)
+            {
+                var length = Math.Min(size, ChunkSize);
+                AppendRandomBytesToFile(result, length);
+                size -= length;
+            }
+        }
+
+        private void AppendRandomBytesToFile(TestFile result, int length)
+        {
+            var bytes = new byte[length];
+            random.NextBytes(bytes);
+            using var stream = new FileStream(result.Filename, FileMode.Append);
+            stream.Write(bytes, 0, bytes.Length);
+        }
+    }
+
+    public class TestFile
+    {
+        public TestFile(string filename)
+        {
+            Filename = filename;
+        }
+
+        public string Filename { get; }
+
+        public void AssertIsEqual(TestFile? other)
+        {
+            if (other == null) Assert.Fail("TestFile is null.");
+            if (other == this || other!.Filename == Filename) Assert.Fail("TestFile is compared to itself.");
+
+            using var stream1 = new FileStream(Filename, FileMode.Open, FileAccess.Read);
+            using var stream2 = new FileStream(other.Filename, FileMode.Open, FileAccess.Read);
+
+            var bytes1 = new byte[FileManager.ChunkSize];
+            var bytes2 = new byte[FileManager.ChunkSize];
+
+            var read1 = 0;
+            var read2 = 0;
+
+            while (true)
+            {
+                read1 = stream1.Read(bytes1, 0, FileManager.ChunkSize);
+                read2 = stream2.Read(bytes2, 0, FileManager.ChunkSize);
+
+                if (read1 == 0 && read2 == 0) return;
+                Assert.That(read1, Is.EqualTo(read2), "Files are not of equal length.");
+                CollectionAssert.AreEqual(bytes1, bytes2, "Files are not binary-equal.");
+            }
+        }
+    }
+}

TestCore/K8sManager.cs

@@ -0,0 +1,323 @@
+using k8s;
+using k8s.Models;
+
+namespace CodexDistTests.TestCore
+{
+    public interface IK8sManager
+    {
+        IOnlineCodexNode BringOnline(OfflineCodexNode node);
+    }
+
+    public class K8sManager : IK8sManager
+    {
+        private const string k8sNamespace = "codex-test-namespace";
+        private const string codexDockerImage = "thatbenbierens/nim-codex:sha-c9a62de";
+        private readonly IFileManager fileManager;
+        private int freePort;
+        private int nodeOrderNumber;
+
+        private V1Namespace? activeNamespace;
+        private readonly Dictionary<OnlineCodexNode, ActiveNode> activeNodes = new Dictionary<OnlineCodexNode, ActiveNode>();
+
+        public K8sManager(IFileManager fileManager)
+        {
+            this.fileManager = fileManager;
+            freePort = 30001;
+            nodeOrderNumber = 0;
+        }
+
+        public IOnlineCodexNode BringOnline(OfflineCodexNode node)
+        {
+            var client = CreateClient();
+
+            EnsureTestNamespace(client);
+
+            var activeNode = new ActiveNode(GetFreePort(), GetNodeOrderNumber());
+            var codexNode = new OnlineCodexNode(node, fileManager, activeNode.Port);
+            activeNodes.Add(codexNode, activeNode);
+
+            CreateDeployment(activeNode, client);
+            CreateService(activeNode, client);
+
+            WaitUntilOnline(activeNode, client);
+
+            return codexNode;
+        }
+
+        public IOfflineCodexNode BringOffline(IOnlineCodexNode node)
+        {
+            var client = CreateClient();
+
+            var n = (OnlineCodexNode)node;
+            var activeNode = activeNodes[n];
+            activeNodes.Remove(n);
+
+            var deploymentName = activeNode.Deployment.Name();
+            BringOffline(activeNode, client);
+            WaitUntilOffline(deploymentName, client);
+
+            return n.Origin;
+        }
+
+        public void DeleteAllResources()
+        {
+            var client = CreateClient();
+
+            foreach (var activeNode in activeNodes.Values)
+            {
+                BringOffline(activeNode, client);
+            }
+
+            DeleteNamespace(client);
+
+            WaitUntilZeroPods(client);
+            WaitUntilNamespaceDeleted(client);
+        }
+
+        private void BringOffline(ActiveNode activeNode, Kubernetes client)
+        {
+            DeleteDeployment(activeNode, client);
+            DeleteService(activeNode, client);
+        }
+
+        #region Waiting
+
+        private void WaitUntilOnline(ActiveNode activeNode, Kubernetes client)
+        {
+            while (activeNode.Deployment?.Status.AvailableReplicas == null || activeNode.Deployment.Status.AvailableReplicas != 1)
+            {
+                Timing.WaitForServiceDelay();
+                activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), k8sNamespace);
+            }
+        }
+
+        private void WaitUntilOffline(string deploymentName, Kubernetes client)
+        {
+            var deployment = client.ReadNamespacedDeployment(deploymentName, k8sNamespace);
+            while (deployment != null && deployment.Status.AvailableReplicas > 0)
+            {
+                Timing.WaitForServiceDelay();
+                deployment = client.ReadNamespacedDeployment(deploymentName, k8sNamespace);
+            }
+        }
+
+        private void WaitUntilZeroPods(Kubernetes client)
+        {
+            var pods = client.ListNamespacedPod(k8sNamespace);
+            while (pods.Items.Any())
+            {
+                Timing.WaitForServiceDelay();
+                pods = client.ListNamespacedPod(k8sNamespace);
+            }
+        }
+
+        private void WaitUntilNamespaceDeleted(Kubernetes client)
+        {
+            var namespaces = client.ListNamespace();
+            while (namespaces.Items.Any(n => n.Metadata.Name == k8sNamespace))
+            {
+                Timing.WaitForServiceDelay();
+                namespaces = client.ListNamespace();
+            }
+        }
+
+        #endregion
+
+        #region Service management
+
+        private void CreateService(ActiveNode node, Kubernetes client)
+        {
+            var serviceSpec = new V1Service
+            {
+                ApiVersion = "v1",
+                Metadata = node.GetServiceMetadata(),
+                Spec = new V1ServiceSpec
+                {
+                    Type = "NodePort",
+                    Selector = node.GetSelector(),
+                    Ports = new List<V1ServicePort>
+                    {
+                        new V1ServicePort
+                        {
+                            Protocol = "TCP",
+                            Port = 8080,
+                            TargetPort = node.GetContainerPortName(),
+                            NodePort = node.Port
+                        }
+                    }
+                }
+            };
+
+            node.Service = client.CreateNamespacedService(serviceSpec, k8sNamespace);
+        }
+
+        private void DeleteService(ActiveNode node, Kubernetes client)
+        {
+            if (node.Service == null) return;
+            client.DeleteNamespacedService(node.Service.Name(), k8sNamespace);
+            node.Service = null;
+        }
+
+        #endregion
+
+        #region Deployment management
+
+        private void CreateDeployment(ActiveNode node, Kubernetes client)
+        {
+            var deploymentSpec = new V1Deployment
+            {
+                ApiVersion = "apps/v1",
+                Metadata = node.GetDeploymentMetadata(),
+                Spec = new V1DeploymentSpec
+                {
+                    Replicas = 1,
+                    Selector = new V1LabelSelector
+                    {
+                        MatchLabels = node.GetSelector()
+                    },
+                    Template = new V1PodTemplateSpec
+                    {
+                        Metadata = new V1ObjectMeta
+                        {
+                            Labels = node.GetSelector()
+                        },
+                        Spec = new V1PodSpec
+                        {
+                            Containers = new List<V1Container>
+                            {
+                                new V1Container
+                                {
+                                    Name = node.GetContainerName(),
+                                    Image = codexDockerImage,
+                                    Ports = new List<V1ContainerPort>
+                                    {
+                                        new V1ContainerPort
+                                        {
+                                            ContainerPort = 8080,
+                                            Name = node.GetContainerPortName()
+                                        }
+                                    },
+                                    Env = new List<V1EnvVar>// todo
+                                    {
+                                        new V1EnvVar
+                                        {
+                                            Name = "LOG_LEVEL",
+                                            Value = "WARN"
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            };
+
+            node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, k8sNamespace);
+        }
+
+        private void DeleteDeployment(ActiveNode node, Kubernetes client)
+        {
+            if (node.Deployment == null) return;
+            client.DeleteNamespacedDeployment(node.Deployment.Name(), k8sNamespace);
+            node.Deployment = null;
+        }
+
+        #endregion
+
+        #region Namespace management
+
+        private void EnsureTestNamespace(Kubernetes client)
+        {
+            if (activeNamespace != null) return;
+
+            var namespaceSpec = new V1Namespace
+            {
+                ApiVersion = "v1",
+                Metadata = new V1ObjectMeta
+                {
+                    Name = k8sNamespace,
+                    Labels = new Dictionary<string, string> { { "name", k8sNamespace } }
+                }
+            };
+            activeNamespace = client.CreateNamespace(namespaceSpec);
+        }
+
+        private void DeleteNamespace(Kubernetes client)
+        {
+            if (activeNamespace == null) return;
+            client.DeleteNamespace(activeNamespace.Name());
+        }
+
+        #endregion
+
+        private static Kubernetes CreateClient()
+        {
+            // todo: If the default KubeConfig file does not suffice, change it here:
+            var config = KubernetesClientConfiguration.BuildConfigFromConfigFile();
+            return new Kubernetes(config);
+        }
+
+        private int GetFreePort()
+        {
+            var port = freePort;
+            freePort++;
+            return port;
+        }
+
+        private int GetNodeOrderNumber()
+        {
+            var number = nodeOrderNumber;
+            nodeOrderNumber++;
+            return number;
+        }
+
+        public class ActiveNode
+        {
+            public ActiveNode(int port, int orderNumber)
+            {
+                SelectorName = orderNumber.ToString().PadLeft(6, '0');
+                Port = port;
+            }
+
+            public string SelectorName { get; }
+            public int Port { get; }
+            public V1Deployment? Deployment { get; set; }
+            public V1Service? Service { get; set; }
+
+            public V1ObjectMeta GetServiceMetadata()
+            {
+                return new V1ObjectMeta
+                {
+                    Name = "codex-test-entrypoint-" + SelectorName,
+                    NamespaceProperty = k8sNamespace
+                };
+            }
+
+            public V1ObjectMeta GetDeploymentMetadata()
+            {
+                return new V1ObjectMeta
+                {
+                    Name = "codex-test-node-" + SelectorName,
+                    NamespaceProperty = k8sNamespace
+                };
+            }
+
+            public Dictionary<string, string> GetSelector()
+            {
+                return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + SelectorName } };
+            }
+
+            public string GetContainerPortName()
+            {
+                //Caution, was: "codex-api-port" + SelectorName
+                //but string length causes 'UnprocessableEntity' exception in k8s.
+                return "api-" + SelectorName;
+            }
+
+            public string GetContainerName()
+            {
+                return "codex-test-node";
+            }
+        }
+    }
+}

TestCore/OfflineCodexNode.cs

@@ -0,0 +1,56 @@
+namespace CodexDistTests.TestCore
+{
+    public interface IOfflineCodexNode
+    {
+        IOfflineCodexNode WithLogLevel(CodexLogLevel level);
+        IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node);
+        IOfflineCodexNode WithStorageQuota(int storageQuotaBytes);
+        IOnlineCodexNode BringOnline();
+    }
+
+    public enum CodexLogLevel
+    {
+        Trace,
+        Debug,
+        Info,
+        Warn,
+        Error
+    }
+
+    public class OfflineCodexNode : IOfflineCodexNode
+    {
+        private readonly IK8sManager k8SManager;
+
+        public CodexLogLevel? LogLevel { get; private set; }
+        public IOnlineCodexNode? BootstrapNode { get; private set; }
+        public int? StorageQuota { get; private set; }
+
+        public OfflineCodexNode(IK8sManager k8SManager)
+        {
+            this.k8SManager = k8SManager;
+        }
+
+        public IOnlineCodexNode BringOnline()
+        {
+            return k8SManager.BringOnline(this);
+        }
+
+        public IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node)
+        {
+            BootstrapNode = node;
+            return this;
+        }
+
+        public IOfflineCodexNode WithLogLevel(CodexLogLevel level)
+        {
+            LogLevel = level;
+            return this;
+        }
+
+        public IOfflineCodexNode WithStorageQuota(int storageQuotaBytes)
+        {
+            StorageQuota = storageQuotaBytes;
+            return this;
+        }
+    }
+}

TestCore/OnlineCodexNode.cs

@@ -4,12 +4,24 @@ using System.Net.Http.Headers;
 
 namespace CodexDistTests.TestCore
 {
-    public class CodexNode
+    public interface IOnlineCodexNode
     {
+        CodexDebugResponse GetDebugInfo();
+        ContentId UploadFile(TestFile file, int retryCounter = 0);
+        TestFile? DownloadContent(ContentId contentId);
+    }
+
+    public class OnlineCodexNode : IOnlineCodexNode
+    {
+        private readonly IFileManager fileManager;
         private readonly int port;
 
-        public CodexNode(int port)
+        public OfflineCodexNode Origin { get; }
+
+        public OnlineCodexNode(OfflineCodexNode origin, IFileManager fileManager, int port)
         {
+            Origin = origin;
+            this.fileManager = fileManager;
             this.port = port;
         }
 
@@ -18,20 +30,21 @@ namespace CodexDistTests.TestCore
             return HttpGet<CodexDebugResponse>("debug/info");
         }
 
-        public string UploadFile(string filename, int retryCounter = 0)
+        public ContentId UploadFile(TestFile file, int retryCounter = 0)
         {
             try
             {
                 var url = $"http://127.0.0.1:{port}/api/codex/v1/upload";
                 using var client = GetClient();
 
-                var byteData = File.ReadAllBytes(filename);
+                // Todo: If the file is too large to read into memory, we'll need to rewrite this upload POST to be streaming.
+                var byteData = File.ReadAllBytes(file.Filename);
                 using var content = new ByteArrayContent(byteData);
                 content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
                 var response = Utils.Wait(client.PostAsync(url, content));
 
                 var contentId = Utils.Wait(response.Content.ReadAsStringAsync());
-                return contentId;
+                return new ContentId(contentId);
             }
             catch (Exception exception)
             {
@@ -43,14 +56,20 @@ namespace CodexDistTests.TestCore
                 else
                 {
                     Timing.RetryDelay();
-                    return UploadFile(filename, retryCounter + 1);
+                    return UploadFile(file, retryCounter + 1);
                 }
             }
         }
 
-        public byte[]? DownloadContent(string contentId)
+        public TestFile? DownloadContent(ContentId contentId)
         {
-            return HttpGetBytes("download/" + contentId);
+            // Todo: If the file is too large, rewrite to streaming:
+            var bytes = HttpGetBytes("download/" + contentId.Id);
+            if (bytes == null) return null;
+
+            var file = fileManager.CreateEmptyTestFile();
+            File.WriteAllBytes(file.Filename, bytes);
+            return file;
         }
 
         private byte[]? HttpGetBytes(string endpoint, int retryCounter = 0)
@@ -124,4 +143,14 @@ namespace CodexDistTests.TestCore
         public string version { get; set; } = string.Empty;
         public string revision { get; set; } = string.Empty;
     }
+
+    public class ContentId
+    {
+        public ContentId(string id)
+        {
+            Id = id;
+        }
+        
+        public string Id { get; }
+    }
 }