diff --git a/TestCore/ActiveNode.cs b/TestCore/ActiveNode.cs new file mode 100644 index 0000000..a9892ff --- /dev/null +++ b/TestCore/ActiveNode.cs @@ -0,0 +1,61 @@ +using k8s.Models; + +namespace CodexDistTests.TestCore +{ + public class ActiveNode + { + public ActiveNode(OfflineCodexNode origin, int port, int orderNumber) + { + Origin = origin; + SelectorName = orderNumber.ToString().PadLeft(6, '0'); + Port = port; + } + + public OfflineCodexNode Origin { get; } + public string SelectorName { get; } + public int Port { get; } + public V1Deployment? Deployment { get; set; } + public V1Service? Service { get; set; } + public List ActivePodNames { get; } = new List(); + + public V1ObjectMeta GetServiceMetadata() + { + return new V1ObjectMeta + { + Name = "codex-test-entrypoint-" + SelectorName, + NamespaceProperty = K8sManager.K8sNamespace + }; + } + + public V1ObjectMeta GetDeploymentMetadata() + { + return new V1ObjectMeta + { + Name = "codex-test-node-" + SelectorName, + NamespaceProperty = K8sManager.K8sNamespace + }; + } + + public Dictionary GetSelector() + { + return new Dictionary { { "codex-test-node", "dist-test-" + SelectorName } }; + } + + public string GetContainerPortName() + { + //Caution, was: "codex-api-port" + SelectorName + //but string length causes 'UnprocessableEntity' exception in k8s. + return "api-" + SelectorName; + } + + public string GetContainerName() + { + return "codex-test-node"; + } + + public string Describe() + { + return $"CodexNode{SelectorName}-Port:{Port}-{Origin.Describe()}"; + } + } +} diff --git a/TestCore/CodexAPI.cs b/TestCore/CodexAPI.cs new file mode 100644 index 0000000..bc7351d --- /dev/null +++ b/TestCore/CodexAPI.cs @@ -0,0 +1,17 @@ +namespace CodexDistTests.TestCore +{ + public class CodexDebugResponse + { + public string id { get; set; } = string.Empty; + public string[] addrs { get; set; } = new string[0]; + public string repo { get; set; } = string.Empty; + public string spr { get; set; } = string.Empty; + public CodexDebugVersionResponse codex { get; set; } = new(); + } + + public class CodexDebugVersionResponse + { + public string version { get; set; } = string.Empty; + public string revision { get; set; } = string.Empty; + } +} diff --git a/TestCore/Http.cs b/TestCore/Http.cs new file mode 100644 index 0000000..5be0e1d --- /dev/null +++ b/TestCore/Http.cs @@ -0,0 +1,95 @@ +using Newtonsoft.Json; +using NUnit.Framework; +using System.Net.Http.Headers; + +namespace CodexDistTests.TestCore +{ + public class Http + { + private readonly string ip; + private readonly int port; + private readonly string baseUrl; + + public Http(string ip, int port, string baseUrl) + { + this.ip = ip; + this.port = port; + this.baseUrl = baseUrl; + + if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; + if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; + } + + public T HttpGetJson(string route) + { + return Retry(() => + { + using var client = GetClient(); + var url = GetUrl() + route; + var result = Utils.Wait(client.GetAsync(url)); + var json = Utils.Wait(result.Content.ReadAsStringAsync()); + return JsonConvert.DeserializeObject(json)!; + }); + } + + public string HttpPostStream(string route, Stream stream) + { + return Retry(() => + { + using var client = GetClient(); + var url = GetUrl() + route; + + var content = new StreamContent(stream); + content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + var response = Utils.Wait(client.PostAsync(url, content)); + + return Utils.Wait(response.Content.ReadAsStringAsync()); + }); + } + + public Stream HttpGetStream(string route) + { + return Retry(() => + { + var client = GetClient(); + var url = GetUrl() + route; + + return Utils.Wait(client.GetStreamAsync(url)); + }); + } + + private string GetUrl() + { + return $"http://{ip}:{port}{baseUrl}"; + } + + private static T Retry(Func operation) + { + var retryCounter = 0; + + while (true) + { + try + { + return operation(); + } + catch (Exception exception) + { + retryCounter++; + if (retryCounter > Timing.HttpCallRetryCount()) + { + Assert.Fail(exception.Message); + throw; + } + } + } + } + + private static HttpClient GetClient() + { + var client = new HttpClient(); + client.Timeout = Timing.HttpCallTimeout(); + return client; + } + } +} diff --git a/TestCore/K8sManager.cs b/TestCore/K8sManager.cs index 391ec4c..b939bf8 100644 --- a/TestCore/K8sManager.cs +++ b/TestCore/K8sManager.cs @@ -12,21 +12,17 @@ namespace CodexDistTests.TestCore public class K8sManager : IK8sManager { - private const string k8sNamespace = "codex-test-namespace"; + public const string K8sNamespace = "codex-test-namespace"; private readonly CodexDockerImage dockerImage = new CodexDockerImage(); - private readonly IFileManager fileManager; - private int freePort; - private int nodeOrderNumber; - - private V1Namespace? activeNamespace; + private readonly NumberSource numberSource = new NumberSource(); private readonly Dictionary activeNodes = new Dictionary(); private readonly List knownActivePodNames = new List(); + private readonly IFileManager fileManager; + private V1Namespace? activeNamespace; public K8sManager(IFileManager fileManager) { this.fileManager = fileManager; - freePort = 30001; - nodeOrderNumber = 0; } public IOnlineCodexNode BringOnline(OfflineCodexNode node) @@ -35,7 +31,7 @@ namespace CodexDistTests.TestCore EnsureTestNamespace(client); - var activeNode = new ActiveNode(node, GetFreePort(), GetNodeOrderNumber()); + var activeNode = new ActiveNode(node, numberSource.GetFreePort(), numberSource.GetNodeOrderNumber()); var codexNode = new OnlineCodexNode(this, fileManager, activeNode.Port); activeNodes.Add(codexNode, activeNode); @@ -80,7 +76,7 @@ namespace CodexDistTests.TestCore var nodeDescription = node.Describe(); foreach (var podName in node.ActivePodNames) { - var stream = client.ReadNamespacedPodLog(podName, k8sNamespace); + var stream = client.ReadNamespacedPodLog(podName, K8sNamespace); onLog(node.SelectorName, $"{nodeDescription}:{podName}", stream); } } @@ -98,7 +94,7 @@ namespace CodexDistTests.TestCore { WaitUntil(() => { - activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), k8sNamespace); + activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), K8sNamespace); return activeNode.Deployment?.Status.AvailableReplicas != null && activeNode.Deployment.Status.AvailableReplicas > 0; }); @@ -107,7 +103,7 @@ namespace CodexDistTests.TestCore private void AssignActivePodNames(ActiveNode activeNode, Kubernetes client) { - var pods = client.ListNamespacedPod(k8sNamespace); + var pods = client.ListNamespacedPod(K8sNamespace); var podNames = pods.Items.Select(p => p.Name()); foreach (var podName in podNames) { @@ -123,19 +119,19 @@ namespace CodexDistTests.TestCore { WaitUntil(() => { - var deployment = client.ReadNamespacedDeployment(deploymentName, k8sNamespace); + var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); return deployment == null || deployment.Status.AvailableReplicas == 0; }); } private void WaitUntilZeroPods(Kubernetes client) { - WaitUntil(() => !client.ListNamespacedPod(k8sNamespace).Items.Any()); + WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); } private void WaitUntilNamespaceDeleted(Kubernetes client) { - WaitUntil(() => client.ListNamespace().Items.All(n => n.Metadata.Name != k8sNamespace)); + WaitUntil(() => client.ListNamespace().Items.All(n => n.Metadata.Name != K8sNamespace)); } private void WaitUntil(Func predicate) @@ -182,13 +178,13 @@ namespace CodexDistTests.TestCore } }; - node.Service = client.CreateNamespacedService(serviceSpec, k8sNamespace); + node.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); } private void DeleteService(ActiveNode node, Kubernetes client) { if (node.Service == null) return; - client.DeleteNamespacedService(node.Service.Name(), k8sNamespace); + client.DeleteNamespacedService(node.Service.Name(), K8sNamespace); node.Service = null; } @@ -239,13 +235,13 @@ namespace CodexDistTests.TestCore } }; - node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, k8sNamespace); + node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); } private void DeleteDeployment(ActiveNode node, Kubernetes client) { if (node.Deployment == null) return; - client.DeleteNamespacedDeployment(node.Deployment.Name(), k8sNamespace); + client.DeleteNamespacedDeployment(node.Deployment.Name(), K8sNamespace); node.Deployment = null; } @@ -262,8 +258,8 @@ namespace CodexDistTests.TestCore ApiVersion = "v1", Metadata = new V1ObjectMeta { - Name = k8sNamespace, - Labels = new Dictionary { { "name", k8sNamespace } } + Name = K8sNamespace, + Labels = new Dictionary { { "name", K8sNamespace } } } }; activeNamespace = client.CreateNamespace(namespaceSpec); @@ -292,76 +288,5 @@ namespace CodexDistTests.TestCore activeNodes.Remove(n); return activeNode; } - - private int GetFreePort() - { - var port = freePort; - freePort++; - return port; - } - - private int GetNodeOrderNumber() - { - var number = nodeOrderNumber; - nodeOrderNumber++; - return number; - } - - public class ActiveNode - { - public ActiveNode(OfflineCodexNode origin, int port, int orderNumber) - { - Origin = origin; - SelectorName = orderNumber.ToString().PadLeft(6, '0'); - Port = port; - } - - public OfflineCodexNode Origin { get; } - public string SelectorName { get; } - public int Port { get; } - public V1Deployment? Deployment { get; set; } - public V1Service? Service { get; set; } - public List ActivePodNames { get; } = new List(); - - public V1ObjectMeta GetServiceMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-entrypoint-" + SelectorName, - NamespaceProperty = k8sNamespace - }; - } - - public V1ObjectMeta GetDeploymentMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-node-" + SelectorName, - NamespaceProperty = k8sNamespace - }; - } - - public Dictionary GetSelector() - { - return new Dictionary { { "codex-test-node", "dist-test-" + SelectorName } }; - } - - public string GetContainerPortName() - { - //Caution, was: "codex-api-port" + SelectorName - //but string length causes 'UnprocessableEntity' exception in k8s. - return "api-" + SelectorName; - } - - public string GetContainerName() - { - return "codex-test-node"; - } - - public string Describe() - { - return $"CodexNode{SelectorName}-Port:{Port}-{Origin.Describe()}"; - } - } } } diff --git a/TestCore/NumberSource.cs b/TestCore/NumberSource.cs new file mode 100644 index 0000000..0ef5177 --- /dev/null +++ b/TestCore/NumberSource.cs @@ -0,0 +1,28 @@ +namespace CodexDistTests.TestCore +{ + public class NumberSource + { + private int freePort; + private int nodeOrderNumber; + + public NumberSource() + { + freePort = 30001; + nodeOrderNumber = 0; + } + + public int GetFreePort() + { + var port = freePort; + freePort++; + return port; + } + + public int GetNodeOrderNumber() + { + var number = nodeOrderNumber; + nodeOrderNumber++; + return number; + } + } +} diff --git a/TestCore/OnlineCodexNode.cs b/TestCore/OnlineCodexNode.cs index a25aed1..a33150c 100644 --- a/TestCore/OnlineCodexNode.cs +++ b/TestCore/OnlineCodexNode.cs @@ -1,13 +1,11 @@ -using Newtonsoft.Json; -using NUnit.Framework; -using System.Net.Http.Headers; +using NUnit.Framework; namespace CodexDistTests.TestCore { public interface IOnlineCodexNode { CodexDebugResponse GetDebugInfo(); - ContentId UploadFile(TestFile file, int retryCounter = 0); + ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); IOfflineCodexNode BringOffline(); } @@ -32,126 +30,33 @@ namespace CodexDistTests.TestCore public CodexDebugResponse GetDebugInfo() { - return HttpGet("debug/info"); + return Http().HttpGetJson("debug/info"); } - public ContentId UploadFile(TestFile file, int retryCounter = 0) + public ContentId UploadFile(TestFile file) { - try + using var fileStream = File.OpenRead(file.Filename); + var response = Http().HttpPostStream("upload", fileStream); + if (response.StartsWith("Unable to store block")) { - var url = $"http://127.0.0.1:{port}/api/codex/v1/upload"; - using var client = GetClient(); - - // Todo: If the file is too large to read into memory, we'll need to rewrite this upload POST to be streaming. - var byteData = File.ReadAllBytes(file.Filename); - using var content = new ByteArrayContent(byteData); - content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - var response = Utils.Wait(client.PostAsync(url, content)); - - var contentId = Utils.Wait(response.Content.ReadAsStringAsync()); - if (contentId.StartsWith("Unable to store block")) - { - retryCounter = Timing.HttpCallRetryCount() + 1; - Assert.Fail("Node failed to store block."); - } - return new ContentId(contentId); - } - catch (Exception exception) - { - if (retryCounter > Timing.HttpCallRetryCount()) - { - Assert.Fail(exception.Message); - throw; - } - else - { - Timing.RetryDelay(); - return UploadFile(file, retryCounter + 1); - } + Assert.Fail("Node failed to store block."); } + return new ContentId(response); } public TestFile? DownloadContent(ContentId contentId) { - // Todo: If the file is too large, rewrite to streaming: - var bytes = HttpGetBytes("download/" + contentId.Id); - if (bytes == null) return null; - var file = fileManager.CreateEmptyTestFile(); - File.WriteAllBytes(file.Filename, bytes); + using var fileStream = File.OpenWrite(file.Filename); + using var downloadStream = Http().HttpGetStream("download/" + contentId.Id); + downloadStream.CopyTo(fileStream); return file; } - private byte[]? HttpGetBytes(string endpoint, int retryCounter = 0) + private Http Http() { - try - { - using var client = GetClient(); - var url = $"http://127.0.0.1:{port}/api/codex/v1/" + endpoint; - var result = Utils.Wait(client.GetAsync(url)); - return Utils.Wait(result.Content.ReadAsByteArrayAsync()); - } - catch (Exception exception) - { - if (retryCounter > Timing.HttpCallRetryCount()) - { - Assert.Fail(exception.Message); - return null; - } - else - { - Timing.RetryDelay(); - return HttpGetBytes(endpoint, retryCounter + 1); - } - } + return new Http(ip: "127.0.0.1", port: port, baseUrl: "/api/codex/v1"); } - - private T HttpGet(string endpoint, int retryCounter = 0) - { - try - { - using var client = GetClient(); - var url = $"http://127.0.0.1:{port}/api/codex/v1/" + endpoint; - var result = Utils.Wait(client.GetAsync(url)); - var json = Utils.Wait(result.Content.ReadAsStringAsync()); - return JsonConvert.DeserializeObject(json); - } - catch (Exception exception) - { - if (retryCounter > Timing.HttpCallRetryCount()) - { - Assert.Fail(exception.Message); - throw; - } - else - { - Timing.RetryDelay(); - return HttpGet(endpoint, retryCounter + 1); - } - } - } - - private HttpClient GetClient() - { - var client = new HttpClient(); - client.Timeout = Timing.HttpCallTimeout(); - return client; - } - } - - public class CodexDebugResponse - { - public string id { get; set; } = string.Empty; - public string[] addrs { get; set; } = new string[0]; - public string repo { get; set; } = string.Empty; - public string spr { get; set; } = string.Empty; - public CodexDebugVersionResponse codex { get; set; } = new(); - } - - public class CodexDebugVersionResponse - { - public string version { get; set; } = string.Empty; - public string revision { get; set; } = string.Empty; } public class ContentId