diff --git a/CodexDistTestCore/ActiveNode.cs b/CodexDistTestCore/ActiveNode.cs deleted file mode 100644 index a5df3b7..0000000 --- a/CodexDistTestCore/ActiveNode.cs +++ /dev/null @@ -1,61 +0,0 @@ -using k8s.Models; - -namespace CodexDistTestCore -{ - public class ActiveNode - { - public ActiveNode(OfflineCodexNode origin, int port, int orderNumber) - { - Origin = origin; - SelectorName = orderNumber.ToString().PadLeft(6, '0'); - Port = port; - } - - public OfflineCodexNode Origin { get; } - public string SelectorName { get; } - public int Port { get; } - public V1Deployment? Deployment { get; set; } - public V1Service? Service { get; set; } - public List ActivePodNames { get; } = new List(); - - public V1ObjectMeta GetServiceMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-entrypoint-" + SelectorName, - NamespaceProperty = K8sManager.K8sNamespace - }; - } - - public V1ObjectMeta GetDeploymentMetadata() - { - return new V1ObjectMeta - { - Name = "codex-test-node-" + SelectorName, - NamespaceProperty = K8sManager.K8sNamespace - }; - } - - public Dictionary GetSelector() - { - return new Dictionary { { "codex-test-node", "dist-test-" + SelectorName } }; - } - - public string GetContainerPortName() - { - //Caution, was: "codex-api-port" + SelectorName - //but string length causes 'UnprocessableEntity' exception in k8s. - return "api-" + SelectorName; - } - - public string GetContainerName() - { - return "codex-test-node"; - } - - public string Describe() - { - return $"CodexNode{SelectorName}-Port:{Port}-{Origin.Describe()}"; - } - } -} diff --git a/CodexDistTestCore/CodexDockerImage.cs b/CodexDistTestCore/CodexDockerImage.cs index 42b4137..a7dd353 100644 --- a/CodexDistTestCore/CodexDockerImage.cs +++ b/CodexDistTestCore/CodexDockerImage.cs @@ -14,10 +14,10 @@ namespace CodexDistTestCore return "b20483"; } - public List CreateEnvironmentVariables(OfflineCodexNode node) + public List CreateEnvironmentVariables(OfflineCodexNodes node, CodexNodeContainer environment) { var formatter = new EnvFormatter(); - formatter.Create(node); + formatter.Create(node, environment); return formatter.Result; } @@ -25,8 +25,13 @@ namespace CodexDistTestCore { public List Result { get; } = new List(); - public void Create(OfflineCodexNode node) + public void Create(OfflineCodexNodes node, CodexNodeContainer environment) { + AddVar("API_PORT", environment.ApiPort.ToString()); + AddVar("DATA_DIR", environment.DataDir); + AddVar("DISC_PORT", environment.DiscoveryPort.ToString()); + AddVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{environment.ListenPort}"); + if (node.BootstrapNode != null) { var debugInfo = node.BootstrapNode.GetDebugInfo(); diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs new file mode 100644 index 0000000..fa0e348 --- /dev/null +++ b/CodexDistTestCore/CodexNodeContainer.cs @@ -0,0 +1,75 @@ +namespace CodexDistTestCore +{ + public class CodexNodeContainer + { + public CodexNodeContainer(string name, int servicePort, string servicePortName, int apiPort, string containerPortName, int discoveryPort, int listenPort, string dataDir) + { + Name = name; + ServicePort = servicePort; + ServicePortName = servicePortName; + ApiPort = apiPort; + ContainerPortName = containerPortName; + DiscoveryPort = discoveryPort; + ListenPort = listenPort; + DataDir = dataDir; + } + + public string Name { get; } + public int ServicePort { get; } + public string ServicePortName { get; } + public int ApiPort { get; } + public string ContainerPortName { get; } + public int DiscoveryPort { get; } + public int ListenPort { get; } + public string DataDir { get; } + } + + public class CodexGroupNumberSource + { + private readonly NumberSource codexNodeGroupNumberSource = new NumberSource(0); + private readonly NumberSource groupContainerNameSource = new NumberSource(1); + private readonly NumberSource servicePortSource = new NumberSource(30001); + + public int GetNextCodexNodeGroupNumber() + { + return codexNodeGroupNumberSource.GetNextNumber(); + } + + public string GetNextServicePortName() + { + return $"node{groupContainerNameSource.GetNextNumber()}"; + } + + public int GetNextServicePort() + { + return servicePortSource.GetNextNumber(); + } + } + + public class CodexNodeContainerFactory + { + private readonly NumberSource containerNameSource = new NumberSource(1); + private readonly NumberSource codexPortSource = new NumberSource(8080); + private readonly CodexGroupNumberSource groupContainerFactory; + + public CodexNodeContainerFactory(CodexGroupNumberSource groupContainerFactory) + { + this.groupContainerFactory = groupContainerFactory; + } + + public CodexNodeContainer CreateNext() + { + var n = containerNameSource.GetNextNumber(); + return new CodexNodeContainer( + name: $"codex-node{n}", + servicePort: groupContainerFactory.GetNextServicePort(), + servicePortName: groupContainerFactory.GetNextServicePortName(), + apiPort: codexPortSource.GetNextNumber(), + containerPortName: $"api-{n}", + discoveryPort: codexPortSource.GetNextNumber(), + listenPort: codexPortSource.GetNextNumber(), + dataDir: $"datadir{n}" + ); + } + } +} diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs new file mode 100644 index 0000000..8d8fda5 --- /dev/null +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -0,0 +1,101 @@ +using k8s.Models; +using System.Collections; + +namespace CodexDistTestCore +{ + public interface ICodexNodeGroup : IEnumerable + { + IOfflineCodexNodes BringOffline(); + IOnlineCodexNode this[int index] { get; } + } + + public class CodexNodeGroup : ICodexNodeGroup + { + private readonly IK8sManager k8SManager; + + public CodexNodeGroup(int orderNumber, OfflineCodexNodes origin, IK8sManager k8SManager, OnlineCodexNode[] nodes) + { + OrderNumber = orderNumber; + Origin = origin; + this.k8SManager = k8SManager; + Nodes = nodes; + + foreach (var n in nodes) n.Group = this; + } + + public IOnlineCodexNode this[int index] + { + get + { + return Nodes[index]; + } + } + + public IOfflineCodexNodes BringOffline() + { + return k8SManager.BringOffline(this); + } + + public int OrderNumber { get; } + public OfflineCodexNodes Origin { get; } + public OnlineCodexNode[] Nodes { get; } + public V1Deployment? Deployment { get; set; } + public V1Service? Service { get; set; } + public PodInfo? PodInfo { get; set; } + + public CodexNodeContainer[] GetContainers() + { + return Nodes.Select(n => n.Container).ToArray(); + } + + public IEnumerator GetEnumerator() + { + return Nodes.Cast().GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return Nodes.GetEnumerator(); + } + + public V1ObjectMeta GetServiceMetadata() + { + return new V1ObjectMeta + { + Name = "codex-test-entrypoint-" + OrderNumber, + NamespaceProperty = K8sOperations.K8sNamespace + }; + } + + public V1ObjectMeta GetDeploymentMetadata() + { + return new V1ObjectMeta + { + Name = "codex-test-node-" + OrderNumber, + NamespaceProperty = K8sOperations.K8sNamespace + }; + } + + public Dictionary GetSelector() + { + return new Dictionary { { "codex-test-node", "dist-test-" + OrderNumber } }; + } + + public string Describe() + { + return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}"; + } + } + + public class PodInfo + { + public PodInfo(string name, string ip) + { + Name = name; + Ip = ip; + } + + public string Name { get; } + public string Ip { get; } + } +} diff --git a/CodexDistTestCore/DistTest.cs b/CodexDistTestCore/DistTest.cs index ff51d62..55bd75c 100644 --- a/CodexDistTestCore/DistTest.cs +++ b/CodexDistTestCore/DistTest.cs @@ -5,6 +5,7 @@ namespace CodexDistTestCore [SetUpFixture] public abstract class DistTest { + private TestLog log = null!; private FileManager fileManager = null!; private K8sManager k8sManager = null!; @@ -13,11 +14,22 @@ namespace CodexDistTestCore { // Previous test run may have been interrupted. // Begin by cleaning everything up. - fileManager = new FileManager(); - k8sManager = new K8sManager(fileManager); + log = new TestLog(); + fileManager = new FileManager(log); + k8sManager = new K8sManager(log, fileManager); - k8sManager.DeleteAllResources(); - fileManager.DeleteAllTestFiles(); + try + { + k8sManager.DeleteAllResources(); + fileManager.DeleteAllTestFiles(); + } + catch (Exception ex) + { + GlobalTestFailure.HasFailed = true; + log.Error($"Global setup cleanup failed with: {ex}"); + throw; + } + log.Log("Global setup cleanup successful"); } [SetUp] @@ -29,9 +41,9 @@ namespace CodexDistTestCore } else { - TestLog.BeginTest(); - fileManager = new FileManager(); - k8sManager = new K8sManager(fileManager); + log = new TestLog(); + fileManager = new FileManager(log); + k8sManager = new K8sManager(log, fileManager); } } @@ -40,13 +52,13 @@ namespace CodexDistTestCore { try { - TestLog.EndTest(k8sManager); + log.EndTest(k8sManager); k8sManager.DeleteAllResources(); fileManager.DeleteAllTestFiles(); } catch (Exception ex) { - TestLog.Error("Cleanup failed: " + ex.Message); + log.Error("Cleanup failed: " + ex.Message); GlobalTestFailure.HasFailed = true; } } @@ -56,9 +68,9 @@ namespace CodexDistTestCore return fileManager.GenerateTestFile(size); } - public IOfflineCodexNode SetupCodexNode() + public IOfflineCodexNodes SetupCodexNodes(int numberOfNodes) { - return new OfflineCodexNode(k8sManager); + return new OfflineCodexNodes(k8sManager, numberOfNodes); } } diff --git a/CodexDistTestCore/FileManager.cs b/CodexDistTestCore/FileManager.cs index a040e09..f884b0a 100644 --- a/CodexDistTestCore/FileManager.cs +++ b/CodexDistTestCore/FileManager.cs @@ -15,10 +15,12 @@ namespace CodexDistTestCore private const string Folder = "TestDataFiles"; private readonly Random random = new Random(); private readonly List activeFiles = new List(); + private readonly TestLog log; - public FileManager() + public FileManager(TestLog log) { if (!Directory.Exists(Folder)) Directory.CreateDirectory(Folder); + this.log = log; } public TestFile CreateEmptyTestFile() @@ -26,7 +28,6 @@ namespace CodexDistTestCore var result = new TestFile(Path.Combine(Folder, Guid.NewGuid().ToString() + "_test.bin")); File.Create(result.Filename).Close(); activeFiles.Add(result); - TestLog.Log($"Created test file '{result.Filename}'."); return result; } @@ -34,7 +35,7 @@ namespace CodexDistTestCore { var result = CreateEmptyTestFile(); GenerateFileBytes(result, size); - TestLog.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'."); + log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'."); return result; } @@ -73,6 +74,12 @@ namespace CodexDistTestCore public string Filename { get; } + public long GetFileSize() + { + var info = new FileInfo(Filename); + return info.Length; + } + public void AssertIsEqual(TestFile? other) { if (other == null) Assert.Fail("TestFile is null."); diff --git a/CodexDistTestCore/Http.cs b/CodexDistTestCore/Http.cs index 1cff3c8..fd7e31a 100644 --- a/CodexDistTestCore/Http.cs +++ b/CodexDistTestCore/Http.cs @@ -20,18 +20,22 @@ namespace CodexDistTestCore if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; } - public T HttpGetJson(string route) + public string HttpGetString(string route) { return Retry(() => { using var client = GetClient(); var url = GetUrl() + route; var result = Utils.Wait(client.GetAsync(url)); - var json = Utils.Wait(result.Content.ReadAsStringAsync()); - return JsonConvert.DeserializeObject(json)!; + return Utils.Wait(result.Content.ReadAsStringAsync()); }); } + public T HttpGetJson(string route) + { + return JsonConvert.DeserializeObject(HttpGetString(route))!; + } + public string HttpPostStream(string route, Stream stream) { return Retry(() => @@ -75,6 +79,7 @@ namespace CodexDistTestCore } catch (Exception exception) { + Timing.HttpCallRetryDelay(); retryCounter++; if (retryCounter > Timing.HttpCallRetryCount()) { diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index 5a61217..3530c96 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -1,297 +1,86 @@ -using k8s; -using k8s.Models; -using NUnit.Framework; - -namespace CodexDistTestCore +namespace CodexDistTestCore { public interface IK8sManager { - IOnlineCodexNode BringOnline(OfflineCodexNode node); - IOfflineCodexNode BringOffline(IOnlineCodexNode node); + ICodexNodeGroup BringOnline(OfflineCodexNodes node); + IOfflineCodexNodes BringOffline(ICodexNodeGroup node); } public class K8sManager : IK8sManager { - public const string K8sNamespace = "codex-test-namespace"; - private readonly CodexDockerImage dockerImage = new CodexDockerImage(); - private readonly NumberSource numberSource = new NumberSource(); - private readonly Dictionary activeNodes = new Dictionary(); - private readonly List knownActivePodNames = new List(); + private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource(); + private readonly List onlineCodexNodes = new List(); + private readonly KnownK8sPods knownPods = new KnownK8sPods(); + private readonly TestLog log; private readonly IFileManager fileManager; - public K8sManager(IFileManager fileManager) + public K8sManager(TestLog log, IFileManager fileManager) { + this.log = log; this.fileManager = fileManager; } - public IOnlineCodexNode BringOnline(OfflineCodexNode node) + public ICodexNodeGroup BringOnline(OfflineCodexNodes offline) { - var client = CreateClient(); + var online = CreateOnlineCodexNodes(offline); - EnsureTestNamespace(client); + K8s(k => k.BringOnline(online, offline)); - var activeNode = new ActiveNode(node, numberSource.GetFreePort(), numberSource.GetNodeOrderNumber()); - var codexNode = new OnlineCodexNode(this, fileManager, activeNode.Port); - activeNodes.Add(codexNode, activeNode); + log.Log($"{online.Describe()} online."); - CreateDeployment(activeNode, client, node); - CreateService(activeNode, client); - - WaitUntilOnline(activeNode, client); - TestLog.Log($"{activeNode.Describe()} online."); - - return codexNode; + return online; } - public IOfflineCodexNode BringOffline(IOnlineCodexNode node) + public IOfflineCodexNodes BringOffline(ICodexNodeGroup node) { - var client = CreateClient(); + var online = GetAndRemoveActiveNodeFor(node); - var activeNode = GetAndRemoveActiveNodeFor(node); + K8s(k => k.BringOffline(online)); - var deploymentName = activeNode.Deployment.Name(); - BringOffline(activeNode, client); - WaitUntilOffline(deploymentName, client); - TestLog.Log($"{activeNode.Describe()} offline."); + log.Log($"{online.Describe()} offline."); - return activeNode.Origin; + return online.Origin; } public void DeleteAllResources() { - var client = CreateClient(); - - DeleteNamespace(client); - - WaitUntilZeroPods(client); - WaitUntilNamespaceDeleted(client); + K8s(k => k.DeleteAllResources()); } - public void FetchAllPodsLogs(Action onLog) + public void FetchAllPodsLogs(IPodLogsHandler logHandler) { - var client = CreateClient(); - foreach (var node in activeNodes.Values) - { - var nodeDescription = node.Describe(); - foreach (var podName in node.ActivePodNames) - { - var stream = client.ReadNamespacedPodLog(podName, K8sNamespace); - onLog(node.SelectorName, $"{nodeDescription}:{podName}", stream); - } - } + K8s(k => k.FetchAllPodsLogs(onlineCodexNodes.ToArray(), logHandler)); } - private void BringOffline(ActiveNode activeNode, Kubernetes client) + private CodexNodeGroup CreateOnlineCodexNodes(OfflineCodexNodes offline) { - DeleteDeployment(activeNode, client); - DeleteService(activeNode, client); + var containers = CreateContainers(offline.NumberOfNodes); + var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray(); + var result = new CodexNodeGroup(codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online); + onlineCodexNodes.Add(result); + return result; } - #region Waiting - - private void WaitUntilOnline(ActiveNode activeNode, Kubernetes client) + private CodexNodeContainer[] CreateContainers(int number) { - WaitUntil(() => - { - activeNode.Deployment = client.ReadNamespacedDeployment(activeNode.Deployment.Name(), K8sNamespace); - return activeNode.Deployment?.Status.AvailableReplicas != null && activeNode.Deployment.Status.AvailableReplicas > 0; - }); - - AssignActivePodNames(activeNode, client); + var factory = new CodexNodeContainerFactory(codexGroupNumberSource); + var containers = new List(); + for (var i = 0; i < number; i++) containers.Add(factory.CreateNext()); + return containers.ToArray(); } - private void AssignActivePodNames(ActiveNode activeNode, Kubernetes client) + private CodexNodeGroup GetAndRemoveActiveNodeFor(ICodexNodeGroup node) { - var pods = client.ListNamespacedPod(K8sNamespace); - var podNames = pods.Items.Select(p => p.Name()); - foreach (var podName in podNames) - { - if (!knownActivePodNames.Contains(podName)) - { - knownActivePodNames.Add(podName); - activeNode.ActivePodNames.Add(podName); - } - } + var n = (CodexNodeGroup)node; + onlineCodexNodes.Remove(n); + return n; } - private void WaitUntilOffline(string deploymentName, Kubernetes client) + private void K8s(Action action) { - WaitUntil(() => - { - var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); - return deployment == null || deployment.Status.AvailableReplicas == 0; - }); - } - - private void WaitUntilZeroPods(Kubernetes client) - { - WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); - } - - private void WaitUntilNamespaceDeleted(Kubernetes client) - { - WaitUntil(() => !IsTestNamespaceOnline(client)); - } - - private void WaitUntil(Func predicate) - { - var start = DateTime.UtcNow; - var state = predicate(); - while (!state) - { - if (DateTime.UtcNow - start > Timing.K8sOperationTimeout()) - { - Assert.Fail("K8s operation timed out."); - throw new TimeoutException(); - } - - Timing.WaitForK8sServiceDelay(); - state = predicate(); - } - } - - #endregion - - #region Service management - - private void CreateService(ActiveNode node, Kubernetes client) - { - var serviceSpec = new V1Service - { - ApiVersion = "v1", - Metadata = node.GetServiceMetadata(), - Spec = new V1ServiceSpec - { - Type = "NodePort", - Selector = node.GetSelector(), - Ports = new List - { - new V1ServicePort - { - Protocol = "TCP", - Port = 8080, - TargetPort = node.GetContainerPortName(), - NodePort = node.Port - } - } - } - }; - - node.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); - } - - private void DeleteService(ActiveNode node, Kubernetes client) - { - if (node.Service == null) return; - client.DeleteNamespacedService(node.Service.Name(), K8sNamespace); - node.Service = null; - } - - #endregion - - #region Deployment management - - private void CreateDeployment(ActiveNode node, Kubernetes client, OfflineCodexNode codexNode) - { - var deploymentSpec = new V1Deployment - { - ApiVersion = "apps/v1", - Metadata = node.GetDeploymentMetadata(), - Spec = new V1DeploymentSpec - { - Replicas = 1, - Selector = new V1LabelSelector - { - MatchLabels = node.GetSelector() - }, - Template = new V1PodTemplateSpec - { - Metadata = new V1ObjectMeta - { - Labels = node.GetSelector() - }, - Spec = new V1PodSpec - { - Containers = new List - { - new V1Container - { - Name = node.GetContainerName(), - Image = dockerImage.GetImageTag(), - Ports = new List - { - new V1ContainerPort - { - ContainerPort = 8080, - Name = node.GetContainerPortName() - } - }, - Env = dockerImage.CreateEnvironmentVariables(codexNode) - } - } - } - } - } - }; - - node.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); - } - - private void DeleteDeployment(ActiveNode node, Kubernetes client) - { - if (node.Deployment == null) return; - client.DeleteNamespacedDeployment(node.Deployment.Name(), K8sNamespace); - node.Deployment = null; - } - - #endregion - - #region Namespace management - - private void EnsureTestNamespace(Kubernetes client) - { - if (IsTestNamespaceOnline(client)) return; - - var namespaceSpec = new V1Namespace - { - ApiVersion = "v1", - Metadata = new V1ObjectMeta - { - Name = K8sNamespace, - Labels = new Dictionary { { "name", K8sNamespace } } - } - }; - client.CreateNamespace(namespaceSpec); - } - - private void DeleteNamespace(Kubernetes client) - { - if (IsTestNamespaceOnline(client)) - { - client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0); - } - } - - #endregion - - private static Kubernetes CreateClient() - { - // todo: If the default KubeConfig file does not suffice, change it here: - var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(); - return new Kubernetes(config); - } - - private static bool IsTestNamespaceOnline(Kubernetes client) - { - return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); - } - - private ActiveNode GetAndRemoveActiveNodeFor(IOnlineCodexNode node) - { - var n = (OnlineCodexNode)node; - var activeNode = activeNodes[n]; - activeNodes.Remove(n); - return activeNode; + var k8s = new K8sOperations(knownPods); + action(k8s); + k8s.Close(); } } } diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs new file mode 100644 index 0000000..f9054b8 --- /dev/null +++ b/CodexDistTestCore/K8sOperations.cs @@ -0,0 +1,286 @@ +using k8s; +using k8s.Models; +using NUnit.Framework; + +namespace CodexDistTestCore +{ + public class K8sOperations + { + public const string K8sNamespace = "codex-test-namespace"; + + private readonly CodexDockerImage dockerImage = new CodexDockerImage(); + private readonly Kubernetes client; + private readonly KnownK8sPods knownPods; + + public K8sOperations(KnownK8sPods knownPods) + { + this.knownPods = knownPods; + + // todo: If the default KubeConfig file does not suffice, change it here: + var config = KubernetesClientConfiguration.BuildConfigFromConfigFile(); + client = new Kubernetes(config); + } + + public void Close() + { + client.Dispose(); + } + + public void BringOnline(CodexNodeGroup online, OfflineCodexNodes offline) + { + EnsureTestNamespace(); + + CreateDeployment(online, offline); + CreateService(online); + + WaitUntilOnline(online); + FetchPodInfo(online); + } + + public void BringOffline(CodexNodeGroup online) + { + var deploymentName = online.Deployment.Name(); + DeleteDeployment(online); + DeleteService(online); + WaitUntilOffline(deploymentName); + } + + public void DeleteAllResources() + { + DeleteNamespace(); + + WaitUntilZeroPods(); + WaitUntilNamespaceDeleted(); + } + + public void FetchAllPodsLogs(CodexNodeGroup[] onlines, IPodLogsHandler logHandler) + { + var logNumberSource = new NumberSource(0); + foreach (var online in onlines) + { + foreach (var node in online) + { + WritePodLogs(online, node, logHandler, logNumberSource); + } + } + } + + private void WritePodLogs(CodexNodeGroup online, IOnlineCodexNode node, IPodLogsHandler logHandler, NumberSource logNumberSource) + { + var n = (OnlineCodexNode)node; + var nodeDescription = $"{online.Describe()} contains {n.GetName()}"; + + var stream = client.ReadNamespacedPodLog(online.PodInfo!.Name, K8sNamespace, n.Container.Name); + logHandler.Log(logNumberSource.GetNextNumber(), nodeDescription, stream); + } + + private void FetchPodInfo(CodexNodeGroup online) + { + var pods = client.ListNamespacedPod(K8sNamespace).Items; + + var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); + Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure."); + + var newPod = newPods.Single(); + online.PodInfo = new PodInfo(newPod.Name(), newPod.Status.PodIP); + + Assert.That(!string.IsNullOrEmpty(online.PodInfo.Name), "Invalid pod name received. Test infra failure."); + Assert.That(!string.IsNullOrEmpty(online.PodInfo.Ip), "Invalid pod IP received. Test infra failure."); + + knownPods.Add(newPod.Name()); + } + + #region Waiting + + private void WaitUntilOnline(CodexNodeGroup online) + { + WaitUntil(() => + { + online.Deployment = client.ReadNamespacedDeployment(online.Deployment.Name(), K8sNamespace); + return online.Deployment?.Status.AvailableReplicas != null && online.Deployment.Status.AvailableReplicas > 0; + }); + } + + private void WaitUntilOffline(string deploymentName) + { + WaitUntil(() => + { + var deployment = client.ReadNamespacedDeployment(deploymentName, K8sNamespace); + return deployment == null || deployment.Status.AvailableReplicas == 0; + }); + } + + private void WaitUntilZeroPods() + { + WaitUntil(() => !client.ListNamespacedPod(K8sNamespace).Items.Any()); + } + + private void WaitUntilNamespaceDeleted() + { + WaitUntil(() => !IsTestNamespaceOnline()); + } + + private void WaitUntil(Func predicate) + { + var start = DateTime.UtcNow; + var state = predicate(); + while (!state) + { + if (DateTime.UtcNow - start > Timing.K8sOperationTimeout()) + { + Assert.Fail("K8s operation timed out."); + throw new TimeoutException(); + } + + Timing.WaitForK8sServiceDelay(); + state = predicate(); + } + } + + #endregion + + #region Service management + + private void CreateService(CodexNodeGroup online) + { + var serviceSpec = new V1Service + { + ApiVersion = "v1", + Metadata = online.GetServiceMetadata(), + Spec = new V1ServiceSpec + { + Type = "NodePort", + Selector = online.GetSelector(), + Ports = CreateServicePorts(online) + } + }; + + online.Service = client.CreateNamespacedService(serviceSpec, K8sNamespace); + } + + private List CreateServicePorts(CodexNodeGroup online) + { + var result = new List(); + var containers = online.GetContainers(); + foreach (var container in containers) + { + result.Add(new V1ServicePort + { + Name = container.ServicePortName, + Protocol = "TCP", + Port = container.ApiPort, + TargetPort = container.ContainerPortName, + NodePort = container.ServicePort + }); + } + return result; + } + + private void DeleteService(CodexNodeGroup online) + { + if (online.Service == null) return; + client.DeleteNamespacedService(online.Service.Name(), K8sNamespace); + online.Service = null; + } + + #endregion + + #region Deployment management + + private void CreateDeployment(CodexNodeGroup online, OfflineCodexNodes offline) + { + var deploymentSpec = new V1Deployment + { + ApiVersion = "apps/v1", + Metadata = online.GetDeploymentMetadata(), + Spec = new V1DeploymentSpec + { + Replicas = 1, + Selector = new V1LabelSelector + { + MatchLabels = online.GetSelector() + }, + Template = new V1PodTemplateSpec + { + Metadata = new V1ObjectMeta + { + Labels = online.GetSelector() + }, + Spec = new V1PodSpec + { + Containers = CreateDeploymentContainers(online, offline) + } + } + } + }; + + online.Deployment = client.CreateNamespacedDeployment(deploymentSpec, K8sNamespace); + } + + private List CreateDeploymentContainers(CodexNodeGroup online, OfflineCodexNodes offline) + { + var result = new List(); + var containers = online.GetContainers(); + foreach (var container in containers) + { + result.Add(new V1Container + { + Name = container.Name, + Image = dockerImage.GetImageTag(), + Ports = new List + { + new V1ContainerPort + { + ContainerPort = container.ApiPort, + Name = container.ContainerPortName + } + }, + Env = dockerImage.CreateEnvironmentVariables(offline, container) + }); + } + return result; + } + + private void DeleteDeployment(CodexNodeGroup online) + { + if (online.Deployment == null) return; + client.DeleteNamespacedDeployment(online.Deployment.Name(), K8sNamespace); + online.Deployment = null; + } + + #endregion + + #region Namespace management + + private void EnsureTestNamespace() + { + if (IsTestNamespaceOnline()) return; + + var namespaceSpec = new V1Namespace + { + ApiVersion = "v1", + Metadata = new V1ObjectMeta + { + Name = K8sNamespace, + Labels = new Dictionary { { "name", K8sNamespace } } + } + }; + client.CreateNamespace(namespaceSpec); + } + + private void DeleteNamespace() + { + if (IsTestNamespaceOnline()) + { + client.DeleteNamespace(K8sNamespace, null, null, gracePeriodSeconds: 0); + } + } + + #endregion + + private bool IsTestNamespaceOnline() + { + return client.ListNamespace().Items.Any(n => n.Metadata.Name == K8sNamespace); + } + } +} diff --git a/CodexDistTestCore/KnownK8sPods.cs b/CodexDistTestCore/KnownK8sPods.cs new file mode 100644 index 0000000..940a147 --- /dev/null +++ b/CodexDistTestCore/KnownK8sPods.cs @@ -0,0 +1,17 @@ +namespace CodexDistTestCore +{ + public class KnownK8sPods + { + private readonly List knownActivePodNames = new List(); + + public bool Contains(string name) + { + return knownActivePodNames.Contains(name); + } + + public void Add(string name) + { + knownActivePodNames.Add(name); + } + } +} diff --git a/CodexDistTestCore/NumberSource.cs b/CodexDistTestCore/NumberSource.cs index 3dfa07b..4338610 100644 --- a/CodexDistTestCore/NumberSource.cs +++ b/CodexDistTestCore/NumberSource.cs @@ -2,27 +2,18 @@ { public class NumberSource { - private int freePort; - private int nodeOrderNumber; + private int number; - public NumberSource() + public NumberSource(int start) { - freePort = 30001; - nodeOrderNumber = 0; + number = start; } - public int GetFreePort() + public int GetNextNumber() { - var port = freePort; - freePort++; - return port; - } - - public int GetNodeOrderNumber() - { - var number = nodeOrderNumber; - nodeOrderNumber++; - return number; + var n = number; + number++; + return n; } } } diff --git a/CodexDistTestCore/OfflineCodexNode.cs b/CodexDistTestCore/OfflineCodexNode.cs deleted file mode 100644 index 6eb018f..0000000 --- a/CodexDistTestCore/OfflineCodexNode.cs +++ /dev/null @@ -1,65 +0,0 @@ -namespace CodexDistTestCore -{ - public interface IOfflineCodexNode - { - IOfflineCodexNode WithLogLevel(CodexLogLevel level); - IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node); - IOfflineCodexNode WithStorageQuota(ByteSize storageQuota); - IOnlineCodexNode BringOnline(); - } - - public enum CodexLogLevel - { - Trace, - Debug, - Info, - Warn, - Error - } - - public class OfflineCodexNode : IOfflineCodexNode - { - private readonly IK8sManager k8SManager; - - public CodexLogLevel? LogLevel { get; private set; } - public IOnlineCodexNode? BootstrapNode { get; private set; } - public ByteSize? StorageQuota { get; private set; } - - public OfflineCodexNode(IK8sManager k8SManager) - { - this.k8SManager = k8SManager; - } - - public IOnlineCodexNode BringOnline() - { - return k8SManager.BringOnline(this); - } - - public IOfflineCodexNode WithBootstrapNode(IOnlineCodexNode node) - { - BootstrapNode = node; - return this; - } - - public IOfflineCodexNode WithLogLevel(CodexLogLevel level) - { - LogLevel = level; - return this; - } - - public IOfflineCodexNode WithStorageQuota(ByteSize storageQuota) - { - StorageQuota = storageQuota; - return this; - } - - public string Describe() - { - var result = ""; - if (LogLevel != null) result += $"LogLevel={LogLevel},"; - if (BootstrapNode != null) result += "BootstrapNode=set,"; - if (StorageQuota != null) result += $"StorageQuote={StorageQuota.SizeInBytes},"; - return result; - } - } -} diff --git a/CodexDistTestCore/OfflineCodexNodes.cs b/CodexDistTestCore/OfflineCodexNodes.cs new file mode 100644 index 0000000..1b820e2 --- /dev/null +++ b/CodexDistTestCore/OfflineCodexNodes.cs @@ -0,0 +1,71 @@ +namespace CodexDistTestCore +{ + public interface IOfflineCodexNodes + { + IOfflineCodexNodes WithLogLevel(CodexLogLevel level); + IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node); + IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota); + ICodexNodeGroup BringOnline(); + } + + public enum CodexLogLevel + { + Trace, + Debug, + Info, + Warn, + Error + } + + public class OfflineCodexNodes : IOfflineCodexNodes + { + private readonly IK8sManager k8SManager; + + public int NumberOfNodes { get; } + public CodexLogLevel? LogLevel { get; private set; } + public IOnlineCodexNode? BootstrapNode { get; private set; } + public ByteSize? StorageQuota { get; private set; } + + public OfflineCodexNodes(IK8sManager k8SManager, int numberOfNodes) + { + this.k8SManager = k8SManager; + NumberOfNodes = numberOfNodes; + } + + public ICodexNodeGroup BringOnline() + { + return k8SManager.BringOnline(this); + } + + public IOfflineCodexNodes WithBootstrapNode(IOnlineCodexNode node) + { + BootstrapNode = node; + return this; + } + + public IOfflineCodexNodes WithLogLevel(CodexLogLevel level) + { + LogLevel = level; + return this; + } + + public IOfflineCodexNodes WithStorageQuota(ByteSize storageQuota) + { + StorageQuota = storageQuota; + return this; + } + + public string Describe() + { + var args = string.Join(',', DescribeArgs()); + return $"{NumberOfNodes} CodexNodes with [{args}]"; + } + + private IEnumerable DescribeArgs() + { + if (LogLevel != null) yield return ($"LogLevel={LogLevel}"); + if (BootstrapNode != null) yield return ("BootstrapNode=set"); + if (StorageQuota != null) yield return ($"StorageQuote={StorageQuota.SizeInBytes}"); + } + } +} diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index d6db1af..d783656 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -7,55 +7,106 @@ namespace CodexDistTestCore CodexDebugResponse GetDebugInfo(); ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); - IOfflineCodexNode BringOffline(); + void ConnectToPeer(IOnlineCodexNode node); } public class OnlineCodexNode : IOnlineCodexNode { - private readonly IK8sManager k8SManager; - private readonly IFileManager fileManager; - private readonly int port; + private const string SuccessfullyConnectedMessage = "Successfully connected to peer"; + private const string UploadFailedMessage = "Unable to store block"; - public OnlineCodexNode(IK8sManager k8SManager, IFileManager fileManager, int port) + private readonly TestLog log; + private readonly IFileManager fileManager; + + public OnlineCodexNode(TestLog log, IFileManager fileManager, CodexNodeContainer container) { - this.k8SManager = k8SManager; + this.log = log; this.fileManager = fileManager; - this.port = port; + Container = container; } - public IOfflineCodexNode BringOffline() + public CodexNodeContainer Container { get; } + public CodexNodeGroup Group { get; internal set; } = null!; + + public string GetName() { - return k8SManager.BringOffline(this); + return $"<{Container.Name}>"; } public CodexDebugResponse GetDebugInfo() { - return Http().HttpGetJson("debug/info"); + var response = Http().HttpGetJson("debug/info"); + Log($"Got DebugInfo with id: '{response.id}'."); + return response; } public ContentId UploadFile(TestFile file) { + Log($"Uploading file of size {file.GetFileSize()}..."); using var fileStream = File.OpenRead(file.Filename); var response = Http().HttpPostStream("upload", fileStream); - if (response.StartsWith("Unable to store block")) + if (response.StartsWith(UploadFailedMessage)) { Assert.Fail("Node failed to store block."); } + Log($"Uploaded file. Received contentId: '{response}'."); return new ContentId(response); } public TestFile? DownloadContent(ContentId contentId) { + Log($"Downloading for contentId: '{contentId.Id}'..."); var file = fileManager.CreateEmptyTestFile(); - using var fileStream = File.OpenWrite(file.Filename); - using var downloadStream = Http().HttpGetStream("download/" + contentId.Id); - downloadStream.CopyTo(fileStream); + DownloadToFile(contentId.Id, file); + Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'."); return file; } + public void ConnectToPeer(IOnlineCodexNode node) + { + var peer = (OnlineCodexNode)node; + + Log($"Connecting to peer {peer.GetName()}..."); + var peerInfo = node.GetDebugInfo(); + var peerId = peerInfo.id; + var peerMultiAddress = GetPeerMultiAddress(peer, peerInfo); + + var response = Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); + + Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes."); + Log($"Successfully connected to peer {peer.GetName()}."); + } + + private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) + { + var multiAddress = peerInfo.addrs.First(); + // Todo: Is there a case where First address in list is not the way? + + if (Group == peer.Group) + { + return multiAddress; + } + + // The peer we want to connect is in a different pod. + // We must replace the default IP with the pod IP in the multiAddress. + return multiAddress.Replace("0.0.0.0", peer.Group.PodInfo!.Ip); + } + + private void DownloadToFile(string contentId, TestFile file) + { + using var fileStream = File.OpenWrite(file.Filename); + using var downloadStream = Http().HttpGetStream("download/" + contentId); + downloadStream.CopyTo(fileStream); + } + private Http Http() { - return new Http(ip: "127.0.0.1", port: port, baseUrl: "/api/codex/v1"); + return new Http(ip: "127.0.0.1", port: Container.ServicePort, baseUrl: "/api/codex/v1"); + } + + private void Log(string msg) + { + log.Log($"{GetName()}: {msg}"); } } diff --git a/CodexDistTestCore/PodLogsHandler.cs b/CodexDistTestCore/PodLogsHandler.cs new file mode 100644 index 0000000..d7898f5 --- /dev/null +++ b/CodexDistTestCore/PodLogsHandler.cs @@ -0,0 +1,7 @@ +namespace CodexDistTestCore +{ + public interface IPodLogsHandler + { + void Log(int id, string podDescription, Stream log); + } +} diff --git a/CodexDistTestCore/TestLog.cs b/CodexDistTestCore/TestLog.cs index 3854cd1..95aad2b 100644 --- a/CodexDistTestCore/TestLog.cs +++ b/CodexDistTestCore/TestLog.cs @@ -5,75 +5,94 @@ namespace CodexDistTestCore public class TestLog { public const string LogRoot = "D:/CodexTestLogs"; + private readonly LogFile file; - private static LogFile? file = null; - - public static void Log(string message) + public TestLog() { - file!.Write(message); - } - - public static void Error(string message) - { - Log($"[ERROR] {message}"); - } - - public static void BeginTest() - { - if (file != null) throw new InvalidOperationException("Test is already started!"); - var name = GetTestName(); file = new LogFile(name); Log($"Begin: {name}"); } - public static void EndTest(K8sManager k8sManager) + public void Log(string message) { - if (file == null) throw new InvalidOperationException("No test is started!"); + file.Write(message); + } + public void Error(string message) + { + Log($"[ERROR] {message}"); + } + public void EndTest(K8sManager k8sManager) + { var result = TestContext.CurrentContext.Result; Log($"Finished: {GetTestName()} = {result.Outcome.Status}"); - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + + if (!string.IsNullOrEmpty(result.Message)) { - IncludeFullPodLogging(k8sManager); + Log(result.Message); } - file = null; + if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + { + Log($"{result.StackTrace}"); + + var logWriter = new PodLogWriter(file); + logWriter.IncludeFullPodLogging(k8sManager); + } } private static string GetTestName() { var test = TestContext.CurrentContext.Test; var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); - return $"{className}.{test.MethodName}"; + var args = FormatArguments(test); + return $"{className}.{test.MethodName}{args}"; } - private static void LogRaw(string message, string filename) + private static string FormatArguments(TestContext.TestAdapter test) { - file!.WriteRaw(message, filename); + if (test.Arguments == null || !test.Arguments.Any()) return ""; + return $"[{string.Join(',', test.Arguments)}]"; + } + } + + public class PodLogWriter : IPodLogsHandler + { + private readonly LogFile file; + + public PodLogWriter(LogFile file) + { + this.file = file; } - private static void IncludeFullPodLogging(K8sManager k8sManager) + public void IncludeFullPodLogging(K8sManager k8sManager) { - Log("Full pod logging:"); - k8sManager.FetchAllPodsLogs(WritePodLog); + file.Write("Full pod logging:"); + k8sManager.FetchAllPodsLogs(this); } - private static void WritePodLog(string id, string nodeDescription, Stream stream) + public void Log(int id, string podDescription, Stream log) { - Log($"{nodeDescription} -->> {id}"); - LogRaw(nodeDescription, id); - var reader = new StreamReader(stream); + var logFile = id.ToString().PadLeft(6, '0'); + file.Write($"{podDescription} -->> {logFile}"); + LogRaw(podDescription, logFile); + var reader = new StreamReader(log); var line = reader.ReadLine(); while (line != null) { - LogRaw(line, id); + LogRaw(line, logFile); line = reader.ReadLine(); } } + + private void LogRaw(string message, string filename) + { + file!.WriteRaw(message, filename); + } } public class LogFile diff --git a/CodexDistTestCore/Timing.cs b/CodexDistTestCore/Timing.cs index 26c49b2..7ee0edf 100644 --- a/CodexDistTestCore/Timing.cs +++ b/CodexDistTestCore/Timing.cs @@ -25,9 +25,9 @@ namespace CodexDistTestCore return GetTimes().HttpCallRetryCount(); } - public static void RetryDelay() + public static void HttpCallRetryDelay() { - Utils.Sleep(GetTimes().RetryDelay()); + Utils.Sleep(GetTimes().HttpCallRetryDelay()); } public static void WaitForK8sServiceDelay() @@ -52,7 +52,7 @@ namespace CodexDistTestCore { TimeSpan HttpCallTimeout(); int HttpCallRetryCount(); - TimeSpan RetryDelay(); + TimeSpan HttpCallRetryDelay(); TimeSpan WaitForK8sServiceDelay(); TimeSpan K8sOperationTimeout(); } @@ -69,7 +69,7 @@ namespace CodexDistTestCore return 5; } - public TimeSpan RetryDelay() + public TimeSpan HttpCallRetryDelay() { return TimeSpan.FromSeconds(3); } @@ -97,7 +97,7 @@ namespace CodexDistTestCore return 2; } - public TimeSpan RetryDelay() + public TimeSpan HttpCallRetryDelay() { return TimeSpan.FromMinutes(5); } diff --git a/LongTests/BasicTests/SimpleTests.cs b/LongTests/BasicTests/LargeFileTests.cs similarity index 64% rename from LongTests/BasicTests/SimpleTests.cs rename to LongTests/BasicTests/LargeFileTests.cs index 8c71653..b8ab20e 100644 --- a/LongTests/BasicTests/SimpleTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -4,17 +4,17 @@ using NUnit.Framework; namespace LongTests.BasicTests { [TestFixture] - public class SimpleTests : DistTest + public class LargeFileTests : DistTest { [Test, UseLongTimeouts] public void OneClientLargeFileTest() { - var primary = SetupCodexNode() + var primary = SetupCodexNodes(1) .WithLogLevel(CodexLogLevel.Warn) - .WithStorageQuota(10.GB()) - .BringOnline(); + .WithStorageQuota(20.GB()) + .BringOnline()[0]; - var testFile = GenerateTestFile(1.GB()); + var testFile = GenerateTestFile(10.GB()); var contentId = primary.UploadFile(testFile); diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs new file mode 100644 index 0000000..1e2f62e --- /dev/null +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -0,0 +1,53 @@ +using CodexDistTestCore; +using NUnit.Framework; + +namespace LongTests.BasicTests +{ + public class TestInfraTests : DistTest + { + [Test, UseLongTimeouts] + public void TestInfraShouldHave1000AddressSpacesPerPod() + { + var group = SetupCodexNodes(1000).BringOnline(); + + var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray(); + + Assert.That(nodeIds.Length, Is.EqualTo(nodeIds.Distinct().Count()), + "Not all created nodes provided a unique id."); + } + + [Test, UseLongTimeouts] + public void TestInfraSupportsManyConcurrentPods() + { + for (var i = 0; i < 20; i++) + { + var n = SetupCodexNodes(1).BringOnline()[0]; + + Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().id)); + } + } + + [Test, UseLongTimeouts] + public void DownloadConsistencyTest() + { + var primary = SetupCodexNodes(1) + .WithLogLevel(CodexLogLevel.Trace) + .WithStorageQuota(2.MB()) + .BringOnline()[0]; + + var testFile = GenerateTestFile(1.MB()); + + var contentId = primary.UploadFile(testFile); + + var files = new List(); + for (var i = 0; i < 100; i++) + { + files.Add(primary.DownloadContent(contentId)); + } + + Assert.That(files.All(f => f != null)); + Assert.That(files.All(f => f!.GetFileSize() == testFile.GetFileSize())); + foreach (var file in files) file!.AssertIsEqual(testFile); + } + } +} diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index 2b7b729..c052d64 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -11,7 +11,7 @@ namespace Tests.BasicTests { var dockerImage = new CodexDockerImage(); - var node = SetupCodexNode().BringOnline(); + var node = SetupCodexNodes(1).BringOnline()[0]; var debugInfo = node.GetDebugInfo(); @@ -22,10 +22,10 @@ namespace Tests.BasicTests [Test] public void OneClientTest() { - var primary = SetupCodexNode() - .WithLogLevel(CodexLogLevel.Trace) - .WithStorageQuota(2.MB()) - .BringOnline(); + var primary = SetupCodexNodes(1) + .WithLogLevel(CodexLogLevel.Trace) + .WithStorageQuota(2.MB()) + .BringOnline()[0]; var testFile = GenerateTestFile(1.MB()); @@ -36,26 +36,45 @@ namespace Tests.BasicTests testFile.AssertIsEqual(downloadedFile); } - //[Test] - //public void TwoClientTest() - //{ - // var primary = SetupCodexNode() - // .WithLogLevel(CodexLogLevel.Trace) - // .WithStorageQuota(1024 * 1024 * 2) - // .BringOnline(); + [Test] + public void TwoClientOnePodTest() + { + var group = SetupCodexNodes(2) + .WithLogLevel(CodexLogLevel.Trace) + .WithStorageQuota(2.MB()) + .BringOnline(); - // var secondary = SetupCodexNode() - // .WithLogLevel(CodexLogLevel.Trace) - // .WithBootstrapNode(primary) - // .BringOnline(); + var primary = group[0]; + var secondary = group[1]; - // var testFile = GenerateTestFile(1024 * 1024); + PerformTwoClientTest(primary, secondary); + } - // var contentId = primary.UploadFile(testFile); + [Test] + public void TwoClientTwoPodTest() + { + var primary = SetupCodexNodes(1) + .WithStorageQuota(2.MB()) + .BringOnline()[0]; - // var downloadedFile = secondary.DownloadContent(contentId); + var secondary = SetupCodexNodes(1) + .WithStorageQuota(2.MB()) + .BringOnline()[0]; - // testFile.AssertIsEqual(downloadedFile); - //} + PerformTwoClientTest(primary, secondary); + } + + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) + { + primary.ConnectToPeer(secondary); + + var testFile = GenerateTestFile(1.MB()); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = secondary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } } } diff --git a/codexnode-manifest.yml b/codexnode-manifest.yml index c22d3ba..dafddec 100644 --- a/codexnode-manifest.yml +++ b/codexnode-manifest.yml @@ -1,28 +1,3 @@ -# apiVersion: v1 -# kind: Pod -# metadata: -# name: codex-pod -# spec: -# containers: -# - name: codex-node-container -# image: thatbenbierens/nim-codex:sha-c9a62de -# ports: -# - containerPort: 8080 - -# kind: PersistentVolumeClaim -# apiVersion: v1 -# metadata: -# name: codex-volume -# spec: -# accessModes: -# - ReadWriteOnce -# resources: -# requests: -# storage: 1Gi -# storageClassName: rbd - -# --- - apiVersion: v1 kind: Namespace metadata: @@ -48,21 +23,35 @@ spec: codex-node: dist-test spec: containers: - - name: codex-node - image: thatbenbierens/nim-codex:sha-c9a62de + - name: codex-node1 + image: thatbenbierens/nim-codex:sha-b204837 ports: - containerPort: 8080 - name: codex-api-port + name: api-1 env: - - name: LOG_LEVEL - value: WARN - # volumeMounts: - # - mountPath: /datadir - # name: codex-data - # volumes: - # - name: codex-data - # persistentVolumeClaim: - # claimName: codex-volume + - name: API_PORT + value: "8080" + - name: DATA_DIR + value: datadir1 + - name: DISC_PORT + value: "8081" + - name: LISTEN_ADDRS + value: "/ip4/0.0.0.0/tcp/8082" + - name: codex-node2 + image: thatbenbierens/nim-codex:sha-b204837 + ports: + - containerPort: 8083 + name: api-2 + env: + - name: API_PORT + value: "8083" + - name: DATA_DIR + value: datadir2 + - name: DISC_PORT + value: "8084" + - name: LISTEN_ADDRS + value: "/ip4/0.0.0.0/tcp/8085" + --- apiVersion: v1 @@ -75,7 +64,14 @@ spec: selector: codex-node: dist-test ports: - - protocol: TCP + - name: "node1" + protocol: TCP port: 8080 - targetPort: codex-api-port + targetPort: api-1 nodePort: 30001 + - name: "node2" + protocol: TCP + port: 8083 + targetPort: api-2 + nodePort: 30002 +