From 457951c5615874910e0ca143f62b4c25bf269798 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 19 Mar 2023 10:49:03 +0100 Subject: [PATCH] Cleanup and split up for test supporting methods --- BasicTests/DebugEndpointTests.cs | 29 +- TestCore/DistTest.cs | 139 +------- TestCore/FileManager.cs | 92 +++++ TestCore/K8sManager.cs | 323 ++++++++++++++++++ TestCore/OfflineCodexNode.cs | 56 +++ TestCore/{CodexNode.cs => OnlineCodexNode.cs} | 45 ++- 6 files changed, 550 insertions(+), 134 deletions(-) create mode 100644 TestCore/FileManager.cs create mode 100644 TestCore/K8sManager.cs create mode 100644 TestCore/OfflineCodexNode.cs rename TestCore/{CodexNode.cs => OnlineCodexNode.cs} (71%) diff --git a/BasicTests/DebugEndpointTests.cs b/BasicTests/DebugEndpointTests.cs index 4f8e6a4..a218600 100644 --- a/BasicTests/DebugEndpointTests.cs +++ b/BasicTests/DebugEndpointTests.cs @@ -9,14 +9,35 @@ namespace CodexDistTests.BasicTests [Test] public void GetDebugInfo() { - CreateCodexNode(); + var node = SetupCodexNode().BringOnline(); - var node = GetCodexNode(); var debugInfo = node.GetDebugInfo(); Assert.That(debugInfo.spr, Is.Not.Empty); - - DestroyCodexNode(); } + + //[Test] + //public void TwoClientTest() + //{ + // var primaryNodex = SetupCodexNode() + // .WithLogLevel(CodexLogLevel.Warn) + // .WithStorageQuota(1024 * 1024) + // .BringOnline(); + + // var secondaryNodex = SetupCodexNode() + // .WithBootstrapNode(primaryNodex) + // .BringOnline(); + + // var testFile = GenerateTestFile(1024 * 1024); + + // var contentId = primaryNodex.UploadFile(testFile); + + // var downloadedFile = secondaryNodex.DownloadContent(contentId); + + // testFile.AssertIsEqual(downloadedFile); + + // // Test files are automatically deleted. + // // Online nodes are automatically destroyed. + //} } } diff --git a/TestCore/DistTest.cs b/TestCore/DistTest.cs index 9201a50..326a8f1 100644 --- a/TestCore/DistTest.cs +++ b/TestCore/DistTest.cs @@ -1,139 +1,34 @@ -using k8s; -using k8s.Models; +using NUnit.Framework; namespace CodexDistTests.TestCore { public abstract class DistTest { - private const string k8sNamespace = "codex-test-namespace"; + private FileManager fileManager = null!; + private K8sManager k8sManager = null!; - private V1Namespace? activeNamespace; - private V1Deployment? activeDeployment; - private V1Service? activeService; - - public void CreateCodexNode() + [SetUp] + public void SetUpDistTest() { - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(); - var client = new Kubernetes(config); - - var namespaceSpec = new V1Namespace - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = k8sNamespace, - Labels = new Dictionary { { "name", k8sNamespace } } - } - }; - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = new V1ObjectMeta - { - Name = "codex-demo", - NamespaceProperty = k8sNamespace - }, - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = new Dictionary { { "codex-node", "dist-test" } } - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = new Dictionary { { "codex-node", "dist-test" } } - }, - Spec = new V1PodSpec - { - Containers = new List - { - new V1Container - { - Name = "codex-node", - Image = "thatbenbierens/nim-codex:sha-c9a62de", - Ports = new List - { - new V1ContainerPort - { - ContainerPort = 8080, - Name = "codex-api-port" - } - }, - Env = new List - { - new V1EnvVar - { - Name = "LOG_LEVEL", - Value = "WARN" - } - } - } - } - } - } - } - }; - var serviceSpec = new V1Service - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = "codex-entrypoint", - NamespaceProperty = k8sNamespace - }, - Spec = new V1ServiceSpec - { - Type = "NodePort", - Selector = new Dictionary { { "codex-node", "dist-test" } }, - Ports = new List - { - new V1ServicePort - { - Protocol = "TCP", - Port = 8080, - TargetPort = "codex-api-port", - NodePort = 30001 - } - } - } - }; - - activeNamespace = client.CreateNamespace(namespaceSpec); - activeDeployment = client.CreateNamespacedDeployment(deploymentSpec, k8sNamespace); - activeService = client.CreateNamespacedService(serviceSpec, k8sNamespace); - - // todo: wait until online! - while (activeDeployment.Status.AvailableReplicas == null || activeDeployment.Status.AvailableReplicas != 1) - { - Timing.WaitForServiceDelay(); - activeDeployment = client.ReadNamespacedDeployment(activeDeployment.Name(), k8sNamespace); - } + fileManager = new FileManager(); + k8sManager = new K8sManager(fileManager); } - public CodexNode GetCodexNode() + [TearDown] + public void TearDownDistTest() { - return new CodexNode(30001); // matches service spec. + fileManager.DeleteAllTestFiles(); + k8sManager.DeleteAllResources(); } - public void DestroyCodexNode() + public TestFile GenerateTestFile(int size = 1024) { - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(); - var client = new Kubernetes(config); + return fileManager.GenerateTestFile(size); + } - client.DeleteNamespacedService(activeService.Name(), k8sNamespace); - client.DeleteNamespacedDeployment(activeDeployment.Name(), k8sNamespace); - client.DeleteNamespace(activeNamespace.Name()); - - // todo: wait until terminated! - var pods = client.ListNamespacedPod(k8sNamespace); - while (pods.Items.Any()) - { - Timing.WaitForServiceDelay(); - pods = client.ListNamespacedPod(k8sNamespace); - } + public IOfflineCodexNode SetupCodexNode() + { + return new OfflineCodexNode(k8sManager); } } } diff --git a/TestCore/FileManager.cs b/TestCore/FileManager.cs new file mode 100644 index 0000000..3144929 --- /dev/null +++ b/TestCore/FileManager.cs @@ -0,0 +1,92 @@ +using NUnit.Framework; + +namespace CodexDistTests.TestCore +{ + public interface IFileManager + { + TestFile CreateEmptyTestFile(); + TestFile GenerateTestFile(int size = 1024); + void DeleteAllTestFiles(); + } + + public class FileManager : IFileManager + { + public const int ChunkSize = 1024; + private const string Folder = "TestDataFiles"; + private readonly Random random = new Random(); + private readonly List activeFiles = new List(); + + public TestFile CreateEmptyTestFile() + { + var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin")); + activeFiles.Add(result); + return result; + } + + public TestFile GenerateTestFile(int size = 1024) + { + var result = CreateEmptyTestFile(); + GenerateFileBytes(result, size); + return result; + } + + public void DeleteAllTestFiles() + { + foreach (var file in activeFiles) File.Delete(file.Filename); + activeFiles.Clear(); + } + + private void GenerateFileBytes(TestFile result, int size) + { + while (size > 0) + { + var length = Math.Min(size, ChunkSize); + AppendRandomBytesToFile(result, length); + size -= length; + } + } + + private void AppendRandomBytesToFile(TestFile result, int length) + { + var bytes = new byte[length]; + random.NextBytes(bytes); + using var stream = new FileStream(result.Filename, FileMode.Append); + stream.Write(bytes, 0, bytes.Length); + } + } + + public class TestFile + { + public TestFile(string filename) + { + Filename = filename; + } + + public string Filename { get; } + + public void AssertIsEqual(TestFile? other) + { + if (other == null) Assert.Fail("TestFile is null."); + if (other == this || other!.Filename == Filename) Assert.Fail("TestFile is compared to itself."); + + using var stream1 = new FileStream(Filename, FileMode.Open, FileAccess.Read); + using var stream2 = new FileStream(other.Filename, FileMode.Open, FileAccess.Read); + + var bytes1 = new byte[FileManager.ChunkSize]; + var bytes2 = new byte[FileManager.ChunkSize]; + + var read1 = 0; + var read2 = 0; + + while (true) + { + read1 = stream1.Read(bytes1, 0, FileManager.ChunkSize); + read2 = stream2.Read(bytes2, 0, FileManager.ChunkSize); + + if (read1 == 0 && read2 == 0) return; + Assert.That(read1, Is.EqualTo(read2), "Files are not of equal length."); + CollectionAssert.AreEqual(bytes1, bytes2, "Files are not binary-equal."); + } + } + } +} diff --git a/TestCore/K8sManager.cs b/TestCore/K8sManager.cs new file mode 100644 index 0000000..08b0f0d --- /dev/null +++ b/TestCore/K8sManager.cs @@ -0,0 +1,323 @@ +using k8s; +using k8s.Models; + +namespace CodexDistTests.TestCore +{ + public interface IK8sManager + { + IOnlineCodexNode BringOnline(OfflineCodexNode node); + } + + public class K8sManager : IK8sManager + { + private const string k8sNamespace = "codex-test-namespace"; + private const string codexDockerImage = "thatbenbierens/nim-codex:sha-c9a62de"; + private readonly IFileManager fileManager; + private int freePort; + private int nodeOrderNumber; + + private V1Namespace? activeNamespace; + private readonly Dictionary activeNodes = new Dictionary(); + + public K8sManager(IFileManager fileManager) + { + this.fileManager = fileManager; + freePort = 30001; + nodeOrderNumber = 0; + } + + public IOnlineCodexNode BringOnline(OfflineCodexNode node) + { + var client = CreateClient(); + + EnsureTestNamespace(client); + + var activeNode = new ActiveNode(GetFreePort(), GetNodeOrderNumber()); + var codexNode = new OnlineCodexNode(node, fileManager, activeNode.Port); + activeNodes.Add(codexNode, activeNode); + + CreateDeployment(activeNode, client); + CreateService(activeNode, client); + + WaitUntilOnline(activeNode, client); + + return codexNode; + } + + public IOfflineCodexNode BringOffline(IOnlineCodexNode node) + { + var client = CreateClient(); + + var n = (OnlineCodexNode)node; + var activeNode = activeNodes[n]; + activeNodes.Remove(n); + + var deploymentName = activeNode.Deployment.Name(); + BringOffline(activeNode, client); + WaitUntilOffline(deploymentName, client); + + return n.Origin; + } + + public void DeleteAllResources() + { + var client = CreateClient(); + + foreach (var activeNode in activeNodes.Values) + { + BringOffline(activeNode, client); + } + + DeleteNamespace(client); + + WaitUntilZeroPods(client); + WaitUntilNamespaceDeleted(client); + } + + private void BringOffline(ActiveNode activeNode, Kubernetes client) + { + DeleteDeployment(activeNode, client); + DeleteService(activeNode, client); + } + + #region Waiting + + private void WaitUntilOnline(ActiveNode activeNode, Kubernetes client) + { + while (activeNode.Deployment?.Status.AvailableReplicas == null || activeNode.Deployment.Status.AvailableReplicas != 1) + { + Timing.WaitForServiceDelay(); + activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), k8sNamespace); + } + } + + private void WaitUntilOffline(string deploymentName, Kubernetes client) + { + var deployment = client.ReadNamespacedDeployment(deploymentName, k8sNamespace); + while (deployment != null && deployment.Status.AvailableReplicas > 0) + { + Timing.WaitForServiceDelay(); + deployment = client.ReadNamespacedDeployment(deploymentName, k8sNamespace); + } + } + + private void WaitUntilZeroPods(Kubernetes client) + { + var pods = client.ListNamespacedPod(k8sNamespace); + while (pods.Items.Any()) + { + Timing.WaitForServiceDelay(); + pods = client.ListNamespacedPod(k8sNamespace); + } + } + + private void WaitUntilNamespaceDeleted(Kubernetes client) + { + var namespaces = client.ListNamespace(); + while (namespaces.Items.Any(n => n.Metadata.Name == k8sNamespace)) + { + Timing.WaitForServiceDelay(); + namespaces = client.ListNamespace(); + } + } + + #endregion + + #region Service management + + private void CreateService(ActiveNode node, Kubernetes client) + { + var serviceSpec = new V1Service + { + ApiVersion = "v1", + Metadata = node.GetServiceMetadata(), + Spec = new V1ServiceSpec + { + Type = "NodePort", + Selector = node.GetSelector(), + Ports = new List + { + new V1ServicePort + { + Protocol = "TCP", + Port = 8080, + TargetPort = node.GetContainerPortName(), + NodePort = node.Port + } + } + } + }; + + node.Service = client.CreateNamespacedService(serviceSpec, k8sNamespace); + } + + private void DeleteService(ActiveNode node, Kubernetes client) + { + if (node.Service == null) return; + client.DeleteNamespacedService(node.Service.Name(), k8sNamespace); + node.Service = null; + } + + #endregion + + #region Deployment management + + private void CreateDeployment(ActiveNode node, Kubernetes client) + { + var deploymentSpec = new V1Deployment + { + ApiVersion = "apps/v1", + Metadata = node.GetDeploymentMetadata(), + Spec = new V1DeploymentSpec + { + Replicas = 1, + Selector = new V1LabelSelector + { + MatchLabels = node.GetSelector() + }, + Template = new V1PodTemplateSpec + { + Metadata = new V1ObjectMeta + { + Labels = node.GetSelector() + }, + Spec = new V1PodSpec + { + Containers = new List + { + new V1Container + { + Name = node.GetContainerName(), + Image = codexDockerImage, + Ports = new List + { + new V1ContainerPort + { + ContainerPort = 8080, + Name = node.GetContainerPortName() + } + }, + Env = new List// todo + { + new V1EnvVar + { + Name = "LOG_LEVEL", + Value = "WARN" + } + } + } + } + } + } + } + }; + + node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, k8sNamespace); + } + + private void DeleteDeployment(ActiveNode node, Kubernetes client) + { + if (node.Deployment == null) return; + client.DeleteNamespacedDeployment(node.Deployment.Name(), k8sNamespace); + node.Deployment = null; + } + + #endregion + + #region Namespace management + + private void EnsureTestNamespace(Kubernetes client) + { + if (activeNamespace != null) return; + + var namespaceSpec = new V1Namespace + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = k8sNamespace, + Labels = new Dictionary { { "name", k8sNamespace } } + } + }; + activeNamespace = client.CreateNamespace(namespaceSpec); + } + + private void DeleteNamespace(Kubernetes client) + { + if (activeNamespace == null) return; + client.DeleteNamespace(activeNamespace.Name()); + } + + #endregion + + private static Kubernetes CreateClient() + { + // todo: If the default KubeConfig file does not suffice, change it here: + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(); + return new Kubernetes(config); + } + + private int GetFreePort() + { + var port = freePort; + freePort++; + return port; + } + + private int GetNodeOrderNumber() + { + var number = nodeOrderNumber; + nodeOrderNumber++; + return number; + } + + public class ActiveNode + { + public ActiveNode(int port, int orderNumber) + { + SelectorName = orderNumber.ToString().PadLeft(6, '0'); + Port = port; + } + + public string SelectorName { get; } + public int Port { get; } + public V1Deployment? Deployment { get; set; } + public V1Service? Service { get; set; } + + public V1ObjectMeta GetServiceMetadata() + { + return new V1ObjectMeta + { + Name = "codex-test-entrypoint-" + SelectorName, + NamespaceProperty = k8sNamespace + }; + } + + public V1ObjectMeta GetDeploymentMetadata() + { + return new V1ObjectMeta + { + Name = "codex-test-node-" + SelectorName, + NamespaceProperty = k8sNamespace + }; + } + + public Dictionary GetSelector() + { + return new Dictionary { { "codex-test-node", "dist-test-" + SelectorName } }; + } + + public string GetContainerPortName() + { + //Caution, was: "codex-api-port" + SelectorName + //but string length causes 'UnprocessableEntity' exception in k8s. + return "api-" + SelectorName; + } + + public string GetContainerName() + { + return "codex-test-node"; + } + } + } +} diff --git a/TestCore/OfflineCodexNode.cs b/TestCore/OfflineCodexNode.cs new file mode 100644 index 0000000..1173960 --- /dev/null +++ b/TestCore/OfflineCodexNode.cs @@ -0,0 +1,56 @@ +namespace CodexDistTests.TestCore +{ + public interface IOfflineCodexNode + { + IOfflineCodexNode WithLogLevel(CodexLogLevel level); + IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node); + IOfflineCodexNode WithStorageQuota(int storageQuotaBytes); + IOnlineCodexNode BringOnline(); + } + + public enum CodexLogLevel + { + Trace, + Debug, + Info, + Warn, + Error + } + + public class OfflineCodexNode : IOfflineCodexNode + { + private readonly IK8sManager k8SManager; + + public CodexLogLevel? LogLevel { get; private set; } + public IOnlineCodexNode? BootstrapNode { get; private set; } + public int? StorageQuota { get; private set; } + + public OfflineCodexNode(IK8sManager k8SManager) + { + this.k8SManager = k8SManager; + } + + public IOnlineCodexNode BringOnline() + { + return k8SManager.BringOnline(this); + } + + public IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node) + { + BootstrapNode = node; + return this; + } + + public IOfflineCodexNode WithLogLevel(CodexLogLevel level) + { + LogLevel = level; + return this; + } + + public IOfflineCodexNode WithStorageQuota(int storageQuotaBytes) + { + StorageQuota = storageQuotaBytes; + return this; + } + } +} diff --git a/TestCore/CodexNode.cs b/TestCore/OnlineCodexNode.cs similarity index 71% rename from TestCore/CodexNode.cs rename to TestCore/OnlineCodexNode.cs index 833c7e0..e5f6ee3 100644 --- a/TestCore/CodexNode.cs +++ b/TestCore/OnlineCodexNode.cs @@ -4,12 +4,24 @@ using System.Net.Http.Headers; namespace CodexDistTests.TestCore { - public class CodexNode + public interface IOnlineCodexNode { + CodexDebugResponse GetDebugInfo(); + ContentId UploadFile(TestFile file, int retryCounter = 0); + TestFile? DownloadContent(ContentId contentId); + } + + public class OnlineCodexNode : IOnlineCodexNode + { + private readonly IFileManager fileManager; private readonly int port; - public CodexNode(int port) + public OfflineCodexNode Origin { get; } + + public OnlineCodexNode(OfflineCodexNode origin, IFileManager fileManager, int port) { + Origin = origin; + this.fileManager = fileManager; this.port = port; } @@ -18,20 +30,21 @@ namespace CodexDistTests.TestCore return HttpGet("debug/info"); } - public string UploadFile(string filename, int retryCounter = 0) + public ContentId UploadFile(TestFile file, int retryCounter = 0) { try { var url = $"http://127.0.0.1:{port}/api/codex/v1/upload"; using var client = GetClient(); - var byteData = File.ReadAllBytes(filename); + // Todo: If the file is too large to read into memory, we'll need to rewrite this upload POST to be streaming. + var byteData = File.ReadAllBytes(file.Filename); using var content = new ByteArrayContent(byteData); content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); var response = Utils.Wait(client.PostAsync(url, content)); var contentId = Utils.Wait(response.Content.ReadAsStringAsync()); - return contentId; + return new ContentId(contentId); } catch (Exception exception) { @@ -43,14 +56,20 @@ namespace CodexDistTests.TestCore else { Timing.RetryDelay(); - return UploadFile(filename, retryCounter + 1); + return UploadFile(file, retryCounter + 1); } } } - public byte[]? DownloadContent(string contentId) + public TestFile? DownloadContent(ContentId contentId) { - return HttpGetBytes("download/" + contentId); + // Todo: If the file is too large, rewrite to streaming: + var bytes = HttpGetBytes("download/" + contentId.Id); + if (bytes == null) return null; + + var file = fileManager.CreateEmptyTestFile(); + File.WriteAllBytes(file.Filename, bytes); + return file; } private byte[]? HttpGetBytes(string endpoint, int retryCounter = 0) @@ -124,4 +143,14 @@ namespace CodexDistTests.TestCore public string version { get; set; } = string.Empty; public string revision { get; set; } = string.Empty; } + + public class ContentId + { + public ContentId(string id) + { + Id = id; + } + + public string Id { get; } + } }