diff --git a/DistTestCore/AutoBootstrapDistTest.cs b/DistTestCore/AutoBootstrapDistTest.cs index 2ac0f8d..28e9763 100644 --- a/DistTestCore/AutoBootstrapDistTest.cs +++ b/DistTestCore/AutoBootstrapDistTest.cs @@ -21,7 +21,10 @@ namespace DistTestCore [SetUp] public void SetUpBootstrapNode() { - BootstrapNode = BringOnline(new CodexSetup(1))[0]; + BootstrapNode = BringOnline(new CodexSetup(1) + { + LogLevel = Codex.CodexLogLevel.Trace + })[0]; } protected IOnlineCodexNode BootstrapNode { get; private set; } = null!; diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 5a8d472..292cf46 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -24,7 +24,12 @@ namespace DistTestCore.Codex public CodexDebugPeerResponse GetDebugPeer(string peerId) { - return Http(TimeSpan.FromSeconds(2)).HttpGetJson($"debug/peer/{peerId}"); + return GetDebugPeer(peerId, TimeSpan.FromSeconds(2)); + } + + public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) + { + return Http(timeout).HttpGetJson($"debug/peer/{peerId}"); } public string UploadFile(FileStream fileStream) @@ -62,6 +67,7 @@ namespace DistTestCore.Codex var nodePeerId = debugInfo.id; var nodeName = Container.Name; log.AddStringReplace(nodePeerId, $"___{nodeName}___"); + log.AddStringReplace(debugInfo.table.localNode.nodeId, $"__{nodeName}__"); } catch (Exception e) { @@ -99,6 +105,7 @@ namespace DistTestCore.Codex public class CodexDebugTableNodeResponse { public string nodeId { get; set; } = string.Empty; + public string peerId { get; set; } = string.Empty; public string record { get; set; } = string.Empty; public bool seen { get; set; } } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index ccec842..322fb04 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -11,6 +11,7 @@ namespace DistTestCore string GetName(); CodexDebugResponse GetDebugInfo(); CodexDebugPeerResponse GetDebugPeer(string peerId); + CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout); ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); void ConnectToPeer(IOnlineCodexNode node); @@ -48,7 +49,8 @@ namespace DistTestCore public CodexDebugResponse GetDebugInfo() { var debugInfo = CodexAccess.GetDebugInfo(); - Log($"Got DebugInfo with id: '{debugInfo.id}'."); + var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId)); + Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}"); return debugInfo; } @@ -57,6 +59,11 @@ namespace DistTestCore return CodexAccess.GetDebugPeer(peerId); } + public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) + { + return CodexAccess.GetDebugPeer(peerId, timeout); + } + public ContentId UploadFile(TestFile file) { Log($"Uploading file of size {file.GetFileSize()}..."); diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index a2f3ed4..ad9b5c9 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -1,5 +1,7 @@ using DistTestCore; +using DistTestCore.Codex; using NUnit.Framework; +using Utils; namespace Tests.PeerDiscoveryTests { @@ -32,14 +34,24 @@ namespace Tests.PeerDiscoveryTests [TestCase(3, 3)] [TestCase(3, 5)] [TestCase(3, 10)] + [TestCase(5, 10)] + [TestCase(3, 20)] + [TestCase(5, 20)] public void StagedVariableNodes(int numberOfNodes, int numberOfStages) { for (var i = 0; i < numberOfStages; i++) { - SetupCodexNodes(numberOfNodes); + SetupCodexNodes(numberOfNodes, s => s.WithLogLevel(CodexLogLevel.Trace)); AssertAllNodesConnected(); } + + // Retain for a while + for (int i = 0; i < 10; i++) + { + Time.Sleep(TimeSpan.FromSeconds(30)); + AssertAllNodesConnected(); + } } private void AssertAllNodesConnected() diff --git a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs index c404fe3..d9b70a6 100644 --- a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs +++ b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs @@ -8,6 +8,8 @@ namespace Tests.PeerDiscoveryTests { public static class PeerTestHelpers { + private static readonly Random random = new Random(); + public static void AssertFullyConnected(IEnumerable nodes, BaseLog? log = null) { AssertFullyConnected(log, nodes.ToArray()); @@ -31,7 +33,7 @@ namespace Tests.PeerDiscoveryTests private static void RetryWhilePairs(List pairs, Action action) { - var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2); + var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5); while (pairs.Any() && (timeout > DateTime.UtcNow)) { action(); @@ -52,6 +54,10 @@ namespace Tests.PeerDiscoveryTests pairs.Remove(pair); if (log != null) log.Log(pair.GetMessage()); } + else + { + pair.IncreaseTimeout(); + } } } @@ -90,6 +96,10 @@ namespace Tests.PeerDiscoveryTests public class Pair { + private TimeSpan timeout = TimeSpan.FromSeconds(60); + private TimeSpan aToBTime = TimeSpan.FromSeconds(0); + private TimeSpan bToATime = TimeSpan.FromSeconds(0); + public Pair(Entry a, Entry b) { A = a; @@ -104,11 +114,22 @@ namespace Tests.PeerDiscoveryTests public void Check() { - AKnowsB = Knows(A, B); - BKnowsA = Knows(B, A); + ApplyRandomDelay(); + aToBTime = Measure(() => AKnowsB = Knows(A, B)); + bToATime = Measure(() => BKnowsA = Knows(B, A)); } public string GetMessage() + { + return GetResultMessage() + GetTimePostfix(); + } + + public void IncreaseTimeout() + { + //timeout *= 2; + } + + private string GetResultMessage() { var aName = A.Response.id; var bName = B.Response.id; @@ -128,23 +149,47 @@ namespace Tests.PeerDiscoveryTests return $"{aName} and {bName} don't know each other."; } - private static bool Knows(Entry a, Entry b) + private string GetTimePostfix() { - var peerId = b.Response.id; + var aName = A.Response.id; + var bName = B.Response.id; - try + return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)"; + } + + private static void ApplyRandomDelay() + { + // Calling all the nodes all at the same time is not exactly nice. + Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 100))); + } + + private static TimeSpan Measure(Action action) + { + var start = DateTime.UtcNow; + action(); + return DateTime.UtcNow - start; + } + + private bool Knows(Entry a, Entry b) + { + lock (a) { - var response = a.Node.GetDebugPeer(peerId); - if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any()) + var peerId = b.Response.id; + + try + { + var response = a.Node.GetDebugPeer(peerId, timeout); + if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any()) + { + return true; + } + } + catch { - return true; } - } - catch - { - } - return false; + return false; + } } } }