Retain for a while in staged variable nodes peer discovery tests

DistTestCore/AutoBootstrapDistTest.cs

@@ -21,7 +21,10 @@ namespace DistTestCore
         [SetUp]
         public void SetUpBootstrapNode()
         {
-            BootstrapNode = BringOnline(new CodexSetup(1))[0];
+            BootstrapNode = BringOnline(new CodexSetup(1)
+            {
+                LogLevel = Codex.CodexLogLevel.Trace
+            })[0];
         }
 
         protected IOnlineCodexNode BootstrapNode { get; private set; } = null!;

DistTestCore/Codex/CodexAccess.cs

@@ -24,7 +24,12 @@ namespace DistTestCore.Codex
 
         public CodexDebugPeerResponse GetDebugPeer(string peerId)
         {
-            return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugPeerResponse>($"debug/peer/{peerId}");
+            return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
+        }
+
+        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
+        {
+            return Http(timeout).HttpGetJson<CodexDebugPeerResponse>($"debug/peer/{peerId}");
         }
 
         public string UploadFile(FileStream fileStream)
@@ -62,6 +67,7 @@ namespace DistTestCore.Codex
                 var nodePeerId = debugInfo.id;
                 var nodeName = Container.Name;
                 log.AddStringReplace(nodePeerId, $"___{nodeName}___");
+                log.AddStringReplace(debugInfo.table.localNode.nodeId, $"__{nodeName}__");
             }
             catch (Exception e)
             {
@@ -99,6 +105,7 @@ namespace DistTestCore.Codex
     public class CodexDebugTableNodeResponse
     {
         public string nodeId { get; set; } = string.Empty;
+        public string peerId { get; set; } = string.Empty;
         public string record { get; set; } = string.Empty;
         public bool seen { get; set; }
     }

DistTestCore/OnlineCodexNode.cs

@@ -11,6 +11,7 @@ namespace DistTestCore
         string GetName();
         CodexDebugResponse GetDebugInfo();
         CodexDebugPeerResponse GetDebugPeer(string peerId);
+        CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout);
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId);
         void ConnectToPeer(IOnlineCodexNode node);
@@ -48,7 +49,8 @@ namespace DistTestCore
         public CodexDebugResponse GetDebugInfo()
         {
             var debugInfo = CodexAccess.GetDebugInfo();
-            Log($"Got DebugInfo with id: '{debugInfo.id}'.");
+            var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
+            Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
             return debugInfo;
         }
 
@@ -57,6 +59,11 @@ namespace DistTestCore
             return CodexAccess.GetDebugPeer(peerId);
         }
 
+        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
+        {
+            return CodexAccess.GetDebugPeer(peerId, timeout);
+        }
+
         public ContentId UploadFile(TestFile file)
         {
             Log($"Uploading file of size {file.GetFileSize()}...");

Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -1,5 +1,7 @@
 using DistTestCore;
+using DistTestCore.Codex;
 using NUnit.Framework;
+using Utils;
 
 namespace Tests.PeerDiscoveryTests
 {
@@ -32,14 +34,24 @@ namespace Tests.PeerDiscoveryTests
         [TestCase(3, 3)]
         [TestCase(3, 5)]
         [TestCase(3, 10)]
+        [TestCase(5, 10)]
+        [TestCase(3, 20)]
+        [TestCase(5, 20)]
         public void StagedVariableNodes(int numberOfNodes, int numberOfStages)
         {
             for (var i = 0; i < numberOfStages; i++)
             {
-                SetupCodexNodes(numberOfNodes);
+                SetupCodexNodes(numberOfNodes, s => s.WithLogLevel(CodexLogLevel.Trace));
 
                 AssertAllNodesConnected();
             }
+
+            // Retain for a while
+            for (int i = 0; i < 10; i++)
+            {
+                Time.Sleep(TimeSpan.FromSeconds(30));
+                AssertAllNodesConnected();
+            }
         }
 
         private void AssertAllNodesConnected()

Tests/PeerDiscoveryTests/PeerTestHelpers.cs

@@ -8,6 +8,8 @@ namespace Tests.PeerDiscoveryTests
 {
     public static class PeerTestHelpers
     {
+        private static readonly Random random = new Random();
+
         public static void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes, BaseLog? log = null)
         {
             AssertFullyConnected(log, nodes.ToArray());
@@ -31,7 +33,7 @@ namespace Tests.PeerDiscoveryTests
 
         private static void RetryWhilePairs(List<Pair> pairs, Action action)
         {
-            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2);
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5);
             while (pairs.Any() && (timeout > DateTime.UtcNow))
             {
                 action();
@@ -52,6 +54,10 @@ namespace Tests.PeerDiscoveryTests
                     pairs.Remove(pair);
                     if (log != null) log.Log(pair.GetMessage());
                 }
+                else
+                {
+                    pair.IncreaseTimeout();
+                }
             }
         }
 
@@ -90,6 +96,10 @@ namespace Tests.PeerDiscoveryTests
 
         public class Pair
         {
+            private TimeSpan timeout = TimeSpan.FromSeconds(60);
+            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
+            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
+
             public Pair(Entry a, Entry b)
             {
                 A = a;
@@ -104,11 +114,22 @@ namespace Tests.PeerDiscoveryTests
 
             public void Check()
             {
-                AKnowsB = Knows(A, B);
+                ApplyRandomDelay();
-                BKnowsA = Knows(B, A);
+                aToBTime = Measure(() => AKnowsB = Knows(A, B));
+                bToATime = Measure(() => BKnowsA = Knows(B, A));
             }
 
             public string GetMessage()
+            {
+                return GetResultMessage() + GetTimePostfix();
+            }
+
+            public void IncreaseTimeout()
+            {
+                //timeout *= 2;
+            }
+
+            private string GetResultMessage()
             {
                 var aName = A.Response.id;
                 var bName = B.Response.id;
@@ -128,23 +149,47 @@ namespace Tests.PeerDiscoveryTests
                 return $"{aName} and {bName} don't know each other.";
             }
 
-            private static bool Knows(Entry a, Entry b)
+            private string GetTimePostfix()
             {
-                var peerId = b.Response.id;
+                var aName = A.Response.id;
+                var bName = B.Response.id;
 
-                try
+                return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
+            }
+
+            private static void ApplyRandomDelay()
+            {
+                // Calling all the nodes all at the same time is not exactly nice.
+                Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 100)));
+            }
+
+            private static TimeSpan Measure(Action action)
+            {
+                var start = DateTime.UtcNow;
+                action();
+                return DateTime.UtcNow - start;
+            }
+
+            private bool Knows(Entry a, Entry b)
+            {
+                lock (a)
                 {
-                    var response = a.Node.GetDebugPeer(peerId);
+                    var peerId = b.Response.id;
-                    if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
+
+                    try
+                    {
+                        var response = a.Node.GetDebugPeer(peerId, timeout);
+                        if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
+                        {
+                            return true;
+                        }
+                    }
+                    catch
                     {
-                        return true;
                     }
-                }
-                catch
-                {
-                }
 
-                return false;
+                    return false;
+                }
             }
         }
     }