Sizable cleanup

TestCore/ActiveNode.cs

@@ -0,0 +1,61 @@
+using k8s.Models;
+
+namespace CodexDistTests.TestCore
+{
+    public class ActiveNode
+    {
+        public ActiveNode(OfflineCodexNode origin, int port, int orderNumber)
+        {
+            Origin = origin;
+            SelectorName = orderNumber.ToString().PadLeft(6, '0');
+            Port = port;
+        }
+
+        public OfflineCodexNode Origin { get; }
+        public string SelectorName { get; }
+        public int Port { get; }
+        public V1Deployment? Deployment { get; set; }
+        public V1Service? Service { get; set; }
+        public List<string> ActivePodNames { get; } = new List<string>();
+
+        public V1ObjectMeta GetServiceMetadata()
+        {
+            return new V1ObjectMeta
+            {
+                Name = "codex-test-entrypoint-" + SelectorName,
+                NamespaceProperty = K8sManager.K8sNamespace
+            };
+        }
+
+        public V1ObjectMeta GetDeploymentMetadata()
+        {
+            return new V1ObjectMeta
+            {
+                Name = "codex-test-node-" + SelectorName,
+                NamespaceProperty = K8sManager.K8sNamespace
+            };
+        }
+
+        public Dictionary<string, string> GetSelector()
+        {
+            return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + SelectorName } };
+        }
+
+        public string GetContainerPortName()
+        {
+            //Caution, was: "codex-api-port" + SelectorName
+            //but string length causes 'UnprocessableEntity' exception in k8s.
+            return "api-" + SelectorName;
+        }
+
+        public string GetContainerName()
+        {
+            return "codex-test-node";
+        }
+
+        public string Describe()
+        {
+            return $"CodexNode{SelectorName}-Port:{Port}-{Origin.Describe()}";
+        }
+    }
+}

TestCore/CodexAPI.cs

@@ -0,0 +1,17 @@
+namespace CodexDistTests.TestCore
+{
+    public class CodexDebugResponse
+    {
+        public string id { get; set; } = string.Empty;
+        public string[] addrs { get; set; } = new string[0];
+        public string repo { get; set; } = string.Empty;
+        public string spr { get; set; } = string.Empty;
+        public CodexDebugVersionResponse codex { get; set; } = new();
+    }
+
+    public class CodexDebugVersionResponse
+    {
+        public string version { get; set; } = string.Empty;
+        public string revision { get; set; } = string.Empty;
+    }
+}

TestCore/Http.cs

@@ -0,0 +1,95 @@
+using Newtonsoft.Json;
+using NUnit.Framework;
+using System.Net.Http.Headers;
+
+namespace CodexDistTests.TestCore
+{
+    public class Http
+    {
+        private readonly string ip;
+        private readonly int port;
+        private readonly string baseUrl;
+
+        public Http(string ip, int port, string baseUrl)
+        {
+            this.ip = ip;
+            this.port = port;
+            this.baseUrl = baseUrl;
+
+            if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
+            if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
+        }
+
+        public T HttpGetJson<T>(string route)
+        {
+            return Retry(() =>
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+                var result = Utils.Wait(client.GetAsync(url));
+                var json = Utils.Wait(result.Content.ReadAsStringAsync());
+                return JsonConvert.DeserializeObject<T>(json)!;
+            });
+        }
+
+        public string HttpPostStream(string route, Stream stream)
+        {
+            return Retry(() => 
+            {
+                using var client = GetClient();
+                var url = GetUrl() + route;
+
+                var content = new StreamContent(stream);
+                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
+                var response = Utils.Wait(client.PostAsync(url, content));
+
+                return Utils.Wait(response.Content.ReadAsStringAsync());
+            });
+        }
+
+        public Stream HttpGetStream(string route)
+        {
+            return Retry(() =>
+            {
+                var client = GetClient();
+                var url = GetUrl() + route;
+
+                return Utils.Wait(client.GetStreamAsync(url));
+            });
+        }
+
+        private string GetUrl()
+        {
+            return $"http://{ip}:{port}{baseUrl}";
+        }
+
+        private static T Retry<T>(Func<T> operation)
+        {
+            var retryCounter = 0;
+
+            while (true)
+            {
+                try
+                {
+                    return operation();
+                }
+                catch (Exception exception)
+                {
+                    retryCounter++;
+                    if (retryCounter > Timing.HttpCallRetryCount())
+                    {
+                        Assert.Fail(exception.Message);
+                        throw;
+                    }
+                }
+            }
+        }
+
+        private static HttpClient GetClient()
+        {
+            var client = new HttpClient();
+            client.Timeout = Timing.HttpCallTimeout();
+            return client;
+        }
+    }
+}

TestCore/K8sManager.cs

@@ -12,21 +12,17 @@ namespace CodexDistTests.TestCore
 
     public class K8sManager : IK8sManager
     {
-        private const string k8sNamespace = "codex-test-namespace";
+        public const string K8sNamespace = "codex-test-namespace";
         private readonly CodexDockerImage dockerImage = new CodexDockerImage();
-        private readonly IFileManager fileManager;
+        private readonly NumberSource numberSource = new NumberSource();
-        private int freePort;
-        private int nodeOrderNumber;
-
-        private V1Namespace? activeNamespace;
         private readonly Dictionary<OnlineCodexNode, ActiveNode> activeNodes = new Dictionary<OnlineCodexNode, ActiveNode>();
         private readonly List<string> knownActivePodNames = new List<string>();
+        private readonly IFileManager fileManager;
+        private V1Namespace? activeNamespace;
 
         public K8sManager(IFileManager fileManager)
         {
             this.fileManager = fileManager;
-            freePort = 30001;
-            nodeOrderNumber = 0;
         }
 
         public IOnlineCodexNode BringOnline(OfflineCodexNode node)
@@ -35,7 +31,7 @@ namespace CodexDistTests.TestCore
 
             EnsureTestNamespace(client);
 
-            var activeNode = new ActiveNode(node, GetFreePort(), GetNodeOrderNumber());
+            var activeNode = new ActiveNode(node, numberSource.GetFreePort(), numberSource.GetNodeOrderNumber());
             var codexNode = new OnlineCodexNode(this, fileManager, activeNode.Port);
             activeNodes.Add(codexNode, activeNode);
 
@@ -80,7 +76,7 @@ namespace CodexDistTests.TestCore
                 var nodeDescription = node.Describe();
                 foreach (var podName in node.ActivePodNames)
                 {
-                    var stream = client.ReadNamespacedPodLog(podName, k8sNamespace);
+                    var stream = client.ReadNamespacedPodLog(podName, K8sNamespace);
                     onLog(node.SelectorName, $"{nodeDescription}:{podName}", stream);
                 }
             }
@@ -98,7 +94,7 @@ namespace CodexDistTests.TestCore
         {
             WaitUntil(() =>
             {
-                activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), k8sNamespace);
+                activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), K8sNamespace);
                 return activeNode.Deployment?.Status.AvailableReplicas != null && activeNode.Deployment.Status.AvailableReplicas > 0;
             });
 
@@ -107,7 +103,7 @@ namespace CodexDistTests.TestCore
 
         private void AssignActivePodNames(ActiveNode activeNode, Kubernetes client)
         {
-            var pods = client.ListNamespacedPod(k8sNamespace);
+            var pods = client.ListNamespacedPod(K8sNamespace);
             var podNames = pods.Items.Select(p => p.Name());
             foreach (var podName in podNames)
             {
@@ -123,19 +119,19 @@ namespace CodexDistTests.TestCore
         {
             WaitUntil(() =>
             {
-                var deployment = client.ReadNamespacedDeployment(deploymentName, k8sNamespace);
+                var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace);
                 return deployment == null || deployment.Status.AvailableReplicas == 0;
             });
         }
 
         private void WaitUntilZeroPods(Kubernetes client)
         {
-            WaitUntil(() => !client.ListNamespacedPod(k8sNamespace).Items.Any());
+            WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any());
         }
 
         private void WaitUntilNamespaceDeleted(Kubernetes client)
         {
-            WaitUntil(() => client.ListNamespace().Items.All(n => n.Metadata.Name != k8sNamespace));
+            WaitUntil(() => client.ListNamespace().Items.All(n => n.Metadata.Name != K8sNamespace));
         }
 
         private void WaitUntil(Func<bool> predicate)
@@ -182,13 +178,13 @@ namespace CodexDistTests.TestCore
                 }
             };
 
-            node.Service = client.CreateNamespacedService(serviceSpec, k8sNamespace);
+            node.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace);
         }
 
         private void DeleteService(ActiveNode node, Kubernetes client)
         {
             if (node.Service == null) return;
-            client.DeleteNamespacedService(node.Service.Name(), k8sNamespace);
+            client.DeleteNamespacedService(node.Service.Name(), K8sNamespace);
             node.Service = null;
         }
 
@@ -239,13 +235,13 @@ namespace CodexDistTests.TestCore
                 }
             };
 
-            node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, k8sNamespace);
+            node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace);
         }
 
         private void DeleteDeployment(ActiveNode node, Kubernetes client)
         {
             if (node.Deployment == null) return;
-            client.DeleteNamespacedDeployment(node.Deployment.Name(), k8sNamespace);
+            client.DeleteNamespacedDeployment(node.Deployment.Name(), K8sNamespace);
             node.Deployment = null;
         }
 
@@ -262,8 +258,8 @@ namespace CodexDistTests.TestCore
                 ApiVersion = "v1",
                 Metadata = new V1ObjectMeta
                 {
-                    Name = k8sNamespace,
+                    Name = K8sNamespace,
-                    Labels = new Dictionary<string, string> { { "name", k8sNamespace } }
+                    Labels = new Dictionary<string, string> { { "name", K8sNamespace } }
                 }
             };
             activeNamespace = client.CreateNamespace(namespaceSpec);
@@ -292,76 +288,5 @@ namespace CodexDistTests.TestCore
             activeNodes.Remove(n);
             return activeNode;
         }
-
-        private int GetFreePort()
-        {
-            var port = freePort;
-            freePort++;
-            return port;
-        }
-
-        private int GetNodeOrderNumber()
-        {
-            var number = nodeOrderNumber;
-            nodeOrderNumber++;
-            return number;
-        }
-
-        public class ActiveNode
-        {
-            public ActiveNode(OfflineCodexNode origin, int port, int orderNumber)
-            {
-                Origin = origin;
-                SelectorName = orderNumber.ToString().PadLeft(6, '0');
-                Port = port;
-            }
-
-            public OfflineCodexNode Origin { get; }
-            public string SelectorName { get; }
-            public int Port { get; }
-            public V1Deployment? Deployment { get; set; }
-            public V1Service? Service { get; set; }
-            public List<string> ActivePodNames { get; } = new List<string>();
-
-            public V1ObjectMeta GetServiceMetadata()
-            {
-                return new V1ObjectMeta
-                {
-                    Name = "codex-test-entrypoint-" + SelectorName,
-                    NamespaceProperty = k8sNamespace
-                };
-            }
-
-            public V1ObjectMeta GetDeploymentMetadata()
-            {
-                return new V1ObjectMeta
-                {
-                    Name = "codex-test-node-" + SelectorName,
-                    NamespaceProperty = k8sNamespace
-                };
-            }
-
-            public Dictionary<string, string> GetSelector()
-            {
-                return new Dictionary<string, string> { { "codex-test-node", "dist-test-" + SelectorName } };
-            }
-
-            public string GetContainerPortName()
-            {
-                //Caution, was: "codex-api-port" + SelectorName
-                //but string length causes 'UnprocessableEntity' exception in k8s.
-                return "api-" + SelectorName;
-            }
-
-            public string GetContainerName()
-            {
-                return "codex-test-node";
-            }
-
-            public string Describe()
-            {
-                return $"CodexNode{SelectorName}-Port:{Port}-{Origin.Describe()}";
-            }
-        }
     }
 }

TestCore/NumberSource.cs

@@ -0,0 +1,28 @@
+namespace CodexDistTests.TestCore
+{
+    public class NumberSource
+    {
+        private int freePort;
+        private int nodeOrderNumber;
+
+        public NumberSource()
+        {
+            freePort = 30001;
+            nodeOrderNumber = 0;
+        }
+
+        public int GetFreePort()
+        {
+            var port = freePort;
+            freePort++;
+            return port;
+        }
+
+        public int GetNodeOrderNumber()
+        {
+            var number = nodeOrderNumber;
+            nodeOrderNumber++;
+            return number;
+        }
+    }
+}

TestCore/OnlineCodexNode.cs

@@ -1,13 +1,11 @@
-using Newtonsoft.Json;
+using NUnit.Framework;
-using NUnit.Framework;
-using System.Net.Http.Headers;
 
 namespace CodexDistTests.TestCore
 {
     public interface IOnlineCodexNode
     {
         CodexDebugResponse GetDebugInfo();
-        ContentId UploadFile(TestFile file, int retryCounter = 0);
+        ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId);
         IOfflineCodexNode BringOffline();
     }
@@ -32,126 +30,33 @@ namespace CodexDistTests.TestCore
 
         public CodexDebugResponse GetDebugInfo()
         {
-            return HttpGet<CodexDebugResponse>("debug/info");
+            return Http().HttpGetJson<CodexDebugResponse>("debug/info");
         }
 
-        public ContentId UploadFile(TestFile file, int retryCounter = 0)
+        public ContentId UploadFile(TestFile file)
         {
-            try
+            using var fileStream = File.OpenRead(file.Filename);
+            var response = Http().HttpPostStream("upload", fileStream);
+            if (response.StartsWith("Unable to store block"))
             {
-                var url = $"http://127.0.0.1:{port}/api/codex/v1/upload";
+                Assert.Fail("Node failed to store block.");
-                using var client = GetClient();
-
-                // Todo: If the file is too large to read into memory, we'll need to rewrite this upload POST to be streaming.
-                var byteData = File.ReadAllBytes(file.Filename);
-                using var content = new ByteArrayContent(byteData);
-                content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
-                var response = Utils.Wait(client.PostAsync(url, content));
-
-                var contentId = Utils.Wait(response.Content.ReadAsStringAsync());
-                if (contentId.StartsWith("Unable to store block"))
-                {
-                    retryCounter = Timing.HttpCallRetryCount() + 1;
-                    Assert.Fail("Node failed to store block.");
-                }
-                return new ContentId(contentId);
-            }
-            catch (Exception exception)
-            {
-                if (retryCounter > Timing.HttpCallRetryCount())
-                {
-                    Assert.Fail(exception.Message);
-                    throw;
-                }
-                else
-                {
-                    Timing.RetryDelay();
-                    return UploadFile(file, retryCounter + 1);
-                }
             }
+            return new ContentId(response);
         }
 
         public TestFile? DownloadContent(ContentId contentId)
         {
-            // Todo: If the file is too large, rewrite to streaming:
-            var bytes = HttpGetBytes("download/" + contentId.Id);
-            if (bytes == null) return null;
-
             var file = fileManager.CreateEmptyTestFile();
-            File.WriteAllBytes(file.Filename, bytes);
+            using var fileStream = File.OpenWrite(file.Filename);
+            using var downloadStream = Http().HttpGetStream("download/" + contentId.Id);
+            downloadStream.CopyTo(fileStream);
             return file;
         }
 
-        private byte[]? HttpGetBytes(string endpoint, int retryCounter = 0)
+        private Http Http()
         {
-            try
+            return new Http(ip: "127.0.0.1", port: port, baseUrl: "/api/codex/v1");
-            {
-                using var client = GetClient();
-                var url = $"http://127.0.0.1:{port}/api/codex/v1/" + endpoint;
-                var result = Utils.Wait(client.GetAsync(url));
-                return Utils.Wait(result.Content.ReadAsByteArrayAsync());
-            }
-            catch (Exception exception)
-            {
-                if (retryCounter > Timing.HttpCallRetryCount())
-                {
-                    Assert.Fail(exception.Message);
-                    return null;
-                }
-                else
-                {
-                    Timing.RetryDelay();
-                    return HttpGetBytes(endpoint, retryCounter + 1);
-                }
-            }
         }
-
-        private T HttpGet<T>(string endpoint, int retryCounter = 0)
-        {
-            try
-            {
-                using var client = GetClient();
-                var url = $"http://127.0.0.1:{port}/api/codex/v1/" + endpoint;
-                var result = Utils.Wait(client.GetAsync(url));
-                var json = Utils.Wait(result.Content.ReadAsStringAsync());
-                return JsonConvert.DeserializeObject<T>(json);
-            }
-            catch (Exception exception)
-            {
-                if (retryCounter > Timing.HttpCallRetryCount())
-                {
-                    Assert.Fail(exception.Message);
-                    throw;
-                }
-                else
-                {
-                    Timing.RetryDelay();
-                    return HttpGet<T>(endpoint, retryCounter + 1);
-                }
-            }
-        }
-
-        private HttpClient GetClient()
-        {
-            var client = new HttpClient();
-            client.Timeout = Timing.HttpCallTimeout();
-            return client;
-        }
-    }
-
-    public class CodexDebugResponse
-    {
-        public string id { get; set; } = string.Empty;
-        public string[] addrs { get; set; } = new string[0];
-        public string repo { get; set; } = string.Empty;
-        public string spr { get; set; } = string.Empty;
-        public CodexDebugVersionResponse codex { get; set; } = new();
-    }
-
-    public class CodexDebugVersionResponse
-    {
-        public string version { get; set; } = string.Empty;
-        public string revision { get; set; } = string.Empty;
     }
 
     public class ContentId