From 8fc573c4b5cd6605de515215d00f74d61812e1e5 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 22 Mar 2023 14:49:01 +0100 Subject: [PATCH] Setting up two-client-two-pod test --- CodexDistTestCore/CodexNodeContainer.cs | 33 ++++++++++++++++++++++--- CodexDistTestCore/CodexNodeGroup.cs | 16 +++++++++++- CodexDistTestCore/K8sManager.cs | 6 ++--- CodexDistTestCore/K8sOperations.cs | 21 ++++++++++------ CodexDistTestCore/OnlineCodexNode.cs | 14 ++++++++--- Tests/BasicTests/SimpleTests.cs | 19 ++++++++++++++ 6 files changed, 91 insertions(+), 18 deletions(-) diff --git a/CodexDistTestCore/CodexNodeContainer.cs b/CodexDistTestCore/CodexNodeContainer.cs index 73f7847..fa0e348 100644 --- a/CodexDistTestCore/CodexNodeContainer.cs +++ b/CodexDistTestCore/CodexNodeContainer.cs @@ -24,19 +24,46 @@ public string DataDir { get; } } + public class CodexGroupNumberSource + { + private readonly NumberSource codexNodeGroupNumberSource = new NumberSource(0); + private readonly NumberSource groupContainerNameSource = new NumberSource(1); + private readonly NumberSource servicePortSource = new NumberSource(30001); + + public int GetNextCodexNodeGroupNumber() + { + return codexNodeGroupNumberSource.GetNextNumber(); + } + + public string GetNextServicePortName() + { + return $"node{groupContainerNameSource.GetNextNumber()}"; + } + + public int GetNextServicePort() + { + return servicePortSource.GetNextNumber(); + } + } + public class CodexNodeContainerFactory { private readonly NumberSource containerNameSource = new NumberSource(1); - private readonly NumberSource servicePortSource = new NumberSource(30001); private readonly NumberSource codexPortSource = new NumberSource(8080); + private readonly CodexGroupNumberSource groupContainerFactory; + + public CodexNodeContainerFactory(CodexGroupNumberSource groupContainerFactory) + { + this.groupContainerFactory = groupContainerFactory; + } public CodexNodeContainer CreateNext() { var n = containerNameSource.GetNextNumber(); return new CodexNodeContainer( name: $"codex-node{n}", - servicePort: servicePortSource.GetNextNumber(), - servicePortName: $"node{n}", + servicePort: groupContainerFactory.GetNextServicePort(), + servicePortName: groupContainerFactory.GetNextServicePortName(), apiPort: codexPortSource.GetNextNumber(), containerPortName: $"api-{n}", discoveryPort: codexPortSource.GetNextNumber(), diff --git a/CodexDistTestCore/CodexNodeGroup.cs b/CodexDistTestCore/CodexNodeGroup.cs index 32c3cd2..8d8fda5 100644 --- a/CodexDistTestCore/CodexNodeGroup.cs +++ b/CodexDistTestCore/CodexNodeGroup.cs @@ -19,6 +19,8 @@ namespace CodexDistTestCore Origin = origin; this.k8SManager = k8SManager; Nodes = nodes; + + foreach (var n in nodes) n.Group = this; } public IOnlineCodexNode this[int index] @@ -39,7 +41,7 @@ namespace CodexDistTestCore public OnlineCodexNode[] Nodes { get; } public V1Deployment? Deployment { get; set; } public V1Service? Service { get; set; } - public string? PodName { get; set; } + public PodInfo? PodInfo { get; set; } public CodexNodeContainer[] GetContainers() { @@ -84,4 +86,16 @@ namespace CodexDistTestCore return $"CodexNodeGroup#{OrderNumber}-{Origin.Describe()}"; } } + + public class PodInfo + { + public PodInfo(string name, string ip) + { + Name = name; + Ip = ip; + } + + public string Name { get; } + public string Ip { get; } + } } diff --git a/CodexDistTestCore/K8sManager.cs b/CodexDistTestCore/K8sManager.cs index cc15a5d..b410b3f 100644 --- a/CodexDistTestCore/K8sManager.cs +++ b/CodexDistTestCore/K8sManager.cs @@ -8,7 +8,7 @@ public class K8sManager : IK8sManager { - private readonly NumberSource onlineCodexNodeOrderNumberSource = new NumberSource(0); + private readonly CodexGroupNumberSource codexGroupNumberSource = new CodexGroupNumberSource(); private readonly List onlineCodexNodes = new List(); private readonly KnownK8sPods knownPods = new KnownK8sPods(); private readonly TestLog log; @@ -56,14 +56,14 @@ { var containers = CreateContainers(offline.NumberOfNodes); var online = containers.Select(c => new OnlineCodexNode(log, fileManager, c)).ToArray(); - var result = new CodexNodeGroup(onlineCodexNodeOrderNumberSource.GetNextNumber(), offline, this, online); + var result = new CodexNodeGroup(codexGroupNumberSource.GetNextCodexNodeGroupNumber(), offline, this, online); onlineCodexNodes.Add(result); return result; } private CodexNodeContainer[] CreateContainers(int number) { - var factory = new CodexNodeContainerFactory(); + var factory = new CodexNodeContainerFactory(codexGroupNumberSource); var containers = new List(); for (var i = 0; i < number; i++) containers.Add(factory.CreateNext()); return containers.ToArray(); diff --git a/CodexDistTestCore/K8sOperations.cs b/CodexDistTestCore/K8sOperations.cs index adea879..2d65fd6 100644 --- a/CodexDistTestCore/K8sOperations.cs +++ b/CodexDistTestCore/K8sOperations.cs @@ -29,7 +29,7 @@ namespace CodexDistTestCore CreateService(online); WaitUntilOnline(online); - AssignActivePodNames(online); + FetchPodInfo(online); } public void BringOffline(CodexNodeGroup online) @@ -65,19 +65,24 @@ namespace CodexDistTestCore var n = (OnlineCodexNode)node; var nodeDescription = $"{online.Describe()} contains {n.GetName()}"; - var stream = client.ReadNamespacedPodLog(online.PodName, K8sNamespace, n.Container.Name); + var stream = client.ReadNamespacedPodLog(online.PodInfo!.Name, K8sNamespace, n.Container.Name); logHandler.Log(logNumberSource.GetNextNumber(), nodeDescription, stream); } - private void AssignActivePodNames(CodexNodeGroup online) + private void FetchPodInfo(CodexNodeGroup online) { - var pods = client.ListNamespacedPod(K8sNamespace); - var podNames = pods.Items.Select(p => p.Name()); + var pods = client.ListNamespacedPod(K8sNamespace).Items; - var newPodNames = podNames.Where(p => !knownPods.Contains(p)).ToArray(); - Assert.That(newPodNames.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure."); + var newPods = pods.Where(p => !knownPods.Contains(p.Name())).ToArray(); + Assert.That(newPods.Length, Is.EqualTo(1), "Expected only 1 pod to be created. Test infra failure."); - online.PodName = newPodNames.Single(); + var newPod = newPods.Single(); + online.PodInfo = new PodInfo(newPod.Name(), newPod.Status.PodIP); + + Assert.That(!string.IsNullOrEmpty(online.PodInfo.Name), "Invalid pod name received. Test infra failure."); + Assert.That(!string.IsNullOrEmpty(online.PodInfo.Ip), "Invalid pod IP received. Test infra failure."); + + knownPods.Add(newPod.Name()); } #region Waiting diff --git a/CodexDistTestCore/OnlineCodexNode.cs b/CodexDistTestCore/OnlineCodexNode.cs index baf304b..d783656 100644 --- a/CodexDistTestCore/OnlineCodexNode.cs +++ b/CodexDistTestCore/OnlineCodexNode.cs @@ -26,6 +26,7 @@ namespace CodexDistTestCore } public CodexNodeContainer Container { get; } + public CodexNodeGroup Group { get; internal set; } = null!; public string GetName() { @@ -78,10 +79,17 @@ namespace CodexDistTestCore private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo) { - // Todo: If peer is in a different pod, we must replace 0.0.0.0 with the address of that pod! - - return peerInfo.addrs.First(); + var multiAddress = peerInfo.addrs.First(); // Todo: Is there a case where First address in list is not the way? + + if (Group == peer.Group) + { + return multiAddress; + } + + // The peer we want to connect is in a different pod. + // We must replace the default IP with the pod IP in the multiAddress. + return multiAddress.Replace("0.0.0.0", peer.Group.PodInfo!.Ip); } private void DownloadToFile(string contentId, TestFile file) diff --git a/Tests/BasicTests/SimpleTests.cs b/Tests/BasicTests/SimpleTests.cs index d780086..c052d64 100644 --- a/Tests/BasicTests/SimpleTests.cs +++ b/Tests/BasicTests/SimpleTests.cs @@ -47,6 +47,25 @@ namespace Tests.BasicTests var primary = group[0]; var secondary = group[1]; + PerformTwoClientTest(primary, secondary); + } + + [Test] + public void TwoClientTwoPodTest() + { + var primary = SetupCodexNodes(1) + .WithStorageQuota(2.MB()) + .BringOnline()[0]; + + var secondary = SetupCodexNodes(1) + .WithStorageQuota(2.MB()) + .BringOnline()[0]; + + PerformTwoClientTest(primary, secondary); + } + + private void PerformTwoClientTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) + { primary.ConnectToPeer(secondary); var testFile = GenerateTestFile(1.MB());