From 38d5b172f48a0dd38d46f528e27a9951ba5e483f Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 11 May 2023 12:44:53 +0200 Subject: [PATCH] successful peer discovery tests --- DistTestCore/Codex/CodexAccess.cs | 33 ++++++-- DistTestCore/CodexStarter.cs | 3 +- DistTestCore/Http.cs | 16 +++- DistTestCore/OnlineCodexNode.cs | 6 ++ DistTestCore/Timing.cs | 2 +- .../LayeredDiscoveryTests.cs | 1 + Tests/PeerDiscoveryTests/PeerTestHelpers.cs | 79 ++++++++++++------- 7 files changed, 97 insertions(+), 43 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 2cee677..00affc2 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -19,7 +19,12 @@ namespace DistTestCore.Codex public CodexDebugResponse GetDebugInfo() { - return Http().HttpGetJson("debug/info"); + return Http(TimeSpan.FromSeconds(2)).HttpGetJson("debug/info"); + } + + public CodexDebugPeerResponse GetDebugPeer(string peerId) + { + return Http().HttpGetJson($"debug/peer/{peerId}"); } public string UploadFile(FileStream fileStream) @@ -42,6 +47,11 @@ namespace DistTestCore.Codex return Http().HttpPostJson($"storage/request/{contentId}", request); } + public string ConnectToPeer(string peerId, string peerMultiAddress) + { + return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); + } + public void EnsureOnline() { try @@ -60,16 +70,11 @@ namespace DistTestCore.Codex } } - private Http Http() + private Http Http(TimeSpan? timeoutOverride = null) { var ip = Container.Pod.Cluster.IP; var port = Container.ServicePorts[0].Number; - return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1"); - } - - public string ConnectToPeer(string peerId, string peerMultiAddress) - { - return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); + return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1", timeoutOverride); } } @@ -124,6 +129,18 @@ namespace DistTestCore.Codex public string revision { get; set; } = string.Empty; } + public class CodexDebugPeerResponse + { + public string peerId { get; set; } = string.Empty; + public long seqNo { get; set; } + public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty(); + } + + public class CodexDebugPeerAddressResponse + { + public string address { get; set; } = string.Empty; + } + public class CodexSalesAvailabilityRequest { public string size { get; set; } = string.Empty; diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index e36ebca..1c49cad 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,6 +1,7 @@ using DistTestCore.Codex; using DistTestCore.Marketplace; using KubernetesWorkflow; +using Logging; namespace DistTestCore { @@ -74,7 +75,7 @@ namespace DistTestCore { var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); RunningGroups.Add(group); - group.EnsureOnline(); + Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true); return group; } diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index bd9f34c..a4feeb8 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -14,15 +14,16 @@ namespace DistTestCore private readonly string ip; private readonly int port; private readonly string baseUrl; + private readonly TimeSpan? timeoutOverride; - public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl) + public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl, TimeSpan? timeoutOverride = null) { this.log = log; this.timeSet = timeSet; this.ip = ip; this.port = port; this.baseUrl = baseUrl; - + this.timeoutOverride = timeoutOverride; if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; } @@ -125,9 +126,18 @@ namespace DistTestCore } private HttpClient GetClient() + { + if (timeoutOverride.HasValue) + { + return GetClient(timeoutOverride.Value); + } + return GetClient(timeSet.HttpCallTimeout()); + } + + private HttpClient GetClient(TimeSpan timeout) { var client = new HttpClient(); - client.Timeout = timeSet.HttpCallTimeout(); + client.Timeout = timeout; return client; } } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 7fb30e1..ccec842 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -10,6 +10,7 @@ namespace DistTestCore { string GetName(); CodexDebugResponse GetDebugInfo(); + CodexDebugPeerResponse GetDebugPeer(string peerId); ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); void ConnectToPeer(IOnlineCodexNode node); @@ -51,6 +52,11 @@ namespace DistTestCore return debugInfo; } + public CodexDebugPeerResponse GetDebugPeer(string peerId) + { + return CodexAccess.GetDebugPeer(peerId); + } + public ContentId UploadFile(TestFile file) { Log($"Uploading file of size {file.GetFileSize()}..."); diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs index e3f7414..cff8b09 100644 --- a/DistTestCore/Timing.cs +++ b/DistTestCore/Timing.cs @@ -26,7 +26,7 @@ namespace DistTestCore public TimeSpan HttpCallRetryTimeout() { - return TimeSpan.FromSeconds(10); + return TimeSpan.FromMinutes(1); } public TimeSpan HttpCallRetryDelay() diff --git a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index 0be9db1..7028b23 100644 --- a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -32,6 +32,7 @@ namespace Tests.PeerDiscoveryTests [TestCase(3)] [TestCase(5)] [TestCase(10)] + [TestCase(20)] public void NodeChainTest(int chainLength) { var node = SetupCodexNode(); diff --git a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs index af2a3ac..f607888 100644 --- a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs +++ b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs @@ -17,14 +17,14 @@ namespace Tests.PeerDiscoveryTests { Time.Retry(() => { - var infos = nodes.Select(n => FetchDebugInfo(n, log)).ToArray(); + var entries = nodes.Select(n => new Entry(n)).ToArray(); var failureMessags = new List(); - for (var x = 0; x < infos.Length; x++) + for (var x = 0; x < entries.Length; x++) { - for (var y = x + 1; y < infos.Length; y++) + for (var y = x + 1; y < entries.Length; y++) { - AssertKnowEachother(failureMessags, infos[x], infos[y], log); + AssertKnowEachother(failureMessags, entries[x], entries[y], log); } } @@ -32,44 +32,63 @@ namespace Tests.PeerDiscoveryTests }); } - private static CodexDebugResponse FetchDebugInfo(IOnlineCodexNode n, BaseLog? log) - { - var info = n.GetDebugInfo(); - - if (log != null) - { - log.AddStringReplace(info.table.localNode.nodeId, $"-<{n.GetName()}>-"); - } - - return info; - } - - private static void AssertKnowEachother(List failureMessags, CodexDebugResponse a, CodexDebugResponse b, BaseLog? log) + private static void AssertKnowEachother(List failureMessags, Entry a, Entry b, BaseLog? log) { AssertKnows(failureMessags, a, b, log); AssertKnows(failureMessags, b, a, log); } - private static void AssertKnows(List failureMessags, CodexDebugResponse a, CodexDebugResponse b, BaseLog? log) + private static void AssertKnows(List failureMessags, Entry a, Entry b, BaseLog? log) { - //var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); - //var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); - var tableNodes = string.Join(",", a.table.nodes.Select(n => n.nodeId)); + var peerId = b.Response.id; - var success = a.table.nodes.Any(n => n.nodeId == b.table.localNode.nodeId); - - if (log != null) + try { - var msg = success ? "PASS" : "FAIL"; - log.Log($"{msg} {a.table.localNode.nodeId} is looking for {b.table.localNode.nodeId} in table-nodes [{tableNodes}]"); + var response = a.Node.GetDebugPeer(peerId); + if (string.IsNullOrEmpty(response.peerId) || !response.addresses.Any()) + { + failureMessags.Add($"{a.Response.id} did not know peer {peerId}"); + } + else if (log != null) + { + log.Log($"{a.Response.id} knows {peerId}."); + } + } + catch (Exception e) + { + failureMessags.Add($"{a.Response.id} was unable to get 'debug/peer/{peerId}'. {e}"); } - //Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); - //Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); - if (!success) + ////var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); + ////var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); + //var tableNodes = string.Join(",", a.table.nodes.Select(n => n.nodeId)); + + //var success = a.table.nodes.Any(n => n.nodeId == b.table.localNode.nodeId); + + //if (log != null) + //{ + // var msg = success ? "PASS" : "FAIL"; + // log.Log($"{msg} {a.table.localNode.nodeId} is looking for {b.table.localNode.nodeId} in table-nodes [{tableNodes}]"); + //} + + ////Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); + ////Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); + //if (!success) + //{ + // failureMessags.Add($"{a.table.localNode.nodeId} was looking for '{b.table.localNode.nodeId}' in table-nodes [{tableNodes}] but it was not found."); + //} + } + + public class Entry + { + public Entry(IOnlineCodexNode node) { - failureMessags.Add($"{a.table.localNode.nodeId} was looking for '{b.table.localNode.nodeId}' in table-nodes [{tableNodes}] but it was not found."); + Node = node; + Response = node.GetDebugInfo(); } + + public IOnlineCodexNode Node { get ; } + public CodexDebugResponse Response { get; } } } }