Better peer discovery result logging. Staged discovery test.

DistTestCore/Codex/CodexAccess.cs

@@ -24,7 +24,7 @@ namespace DistTestCore.Codex
 
         public CodexDebugPeerResponse GetDebugPeer(string peerId)
         {
-            return Http().HttpGetJson<CodexDebugPeerResponse>($"debug/peer/{peerId}");
+            return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugPeerResponse>($"debug/peer/{peerId}");
         }
 
         public string UploadFile(FileStream fileStream)

DistTestCore/Http.cs

@@ -1,6 +1,5 @@
 using Logging;
 using Newtonsoft.Json;
-using NUnit.Framework;
 using System.Net.Http.Headers;
 using System.Net.Http.Json;
 using Utils;
@@ -120,7 +119,6 @@ namespace DistTestCore
             catch (Exception exception)
             {
                 var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}";
-                Assert.Fail(msg);
                 throw new InvalidOperationException(msg, exception);
             }
         }

Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -6,14 +6,6 @@ namespace Tests.PeerDiscoveryTests
     [TestFixture]
     public class PeerDiscoveryTests : AutoBootstrapDistTest
     {
-        [Test]
-        public void TwoNodes()
-        {
-            SetupCodexNode();
-
-            AssertAllNodesConnected();
-        }
-
         [TestCase(2)]
         [TestCase(3)]
         [TestCase(10)]
@@ -37,6 +29,19 @@ namespace Tests.PeerDiscoveryTests
             AssertAllNodesConnected();
         }
 
+        [TestCase(3, 3)]
+        [TestCase(3, 5)]
+        [TestCase(3, 10)]
+        public void StagedVariableNodes(int numberOfNodes, int numberOfStages)
+        {
+            for (var i = 0; i < numberOfStages; i++)
+            {
+                SetupCodexNodes(numberOfNodes);
+
+                AssertAllNodesConnected();
+            }
+        }
+
         private void AssertAllNodesConnected()
         {
             PeerTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes(), GetTestLog());

Tests/PeerDiscoveryTests/PeerTestHelpers.cs

@@ -31,35 +31,57 @@ namespace Tests.PeerDiscoveryTests
 
         private static void AssertKnowEachother(List<string> failureMessags, Entry a, Entry b, BaseLog? log)
         {
-            AssertKnows(failureMessags, a, b, log);
+            var aKnowsB = Knows(a, b);
-            AssertKnows(failureMessags, b, a, log);
+            var bKnowsA = Knows(b, a);
+
+            var message = GetMessage(a, b, aKnowsB, bKnowsA);
+
+            if (log != null) log.Log(message);
+            if (!aKnowsB || !bKnowsA) failureMessags.Add(message);
         }
 
-        private static void AssertKnows(List<string> failureMessags, Entry a, Entry b, BaseLog? log)
+        private static string GetMessage(Entry a, Entry b, bool aKnowsB, bool bKnowsA)
+        {
+            var aName = a.Response.id;
+            var bName = b.Response.id;
+
+            if (aKnowsB && bKnowsA)
+            {
+                return $"{aName} and {bName} know each other.";
+            }
+            if (aKnowsB)
+            {
+                return $"{aName} knows {bName}, but {bName} does not know {aName}";
+            }
+            if (bKnowsA)
+            {
+                return $"{bName} knows {aName}, but {aName} does not know {bName}";
+            }
+            return $"{aName} and {bName} don't know each other.";
+        }
+
+        private static bool Knows(Entry a, Entry b)
         {
             var peerId = b.Response.id;
 
             try
             {
                 var response = GetDebugPeer(a.Node, peerId);
-                if (string.IsNullOrEmpty(response.peerId) || !response.addresses.Any())
+                if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
                 {
-                    failureMessags.Add($"{a.Response.id} did not know peer {peerId}");
+                    return true;
-                }
-                else if (log != null)
-                {
-                    log.Log($"{a.Response.id} knows {peerId}.");
                 }
             }
             catch
             {
-                failureMessags.Add($"{a.Response.id} was unable to get 'debug/peer/{peerId}'.");
             }
+
+            return false;
         }
 
         private static CodexDebugPeerResponse GetDebugPeer(IOnlineCodexNode node, string peerId)
         {
-            return Time.Retry(() => node.GetDebugPeer(peerId));
+            return Time.Retry(() => node.GetDebugPeer(peerId), TimeSpan.FromMinutes(2), TimeSpan.FromSeconds(0.1));
         }
 
         public class Entry