Merge branch 'feature/peer-discovery-tests'

DistTestCore/AutoBootstrapDistTest.cs

@@ -1,9 +1,9 @@
-namespace DistTestCore
+using NUnit.Framework;
+
+namespace DistTestCore
 {
     public class AutoBootstrapDistTest : DistTest
     {
-        private IOnlineCodexNode? bootstrapNode;
-
         public override IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
         {
             throw new Exception("AutoBootstrapDistTest creates and attaches a single boostrap node for you. " +
@@ -12,19 +12,18 @@
 
         public override ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
         {
-            var codexSetup = new CodexSetup(numberOfNodes);
+            var codexSetup = CreateCodexSetup(numberOfNodes);
             setup(codexSetup);
-            codexSetup.WithBootstrapNode(EnsureBootstapNode());
+            codexSetup.WithBootstrapNode(BootstrapNode);
             return BringOnline(codexSetup);
         }
 
-        private IOnlineCodexNode EnsureBootstapNode()
+        [SetUp]
+        public void SetUpBootstrapNode()
         {
-            if (bootstrapNode == null)
+            BootstrapNode = BringOnline(CreateCodexSetup(1))[0];
-            {
-                bootstrapNode = base.SetupCodexBootstrapNode(s => { });
-            }
-            return bootstrapNode;
         }
+
+        protected IOnlineCodexNode BootstrapNode { get; private set; } = null!;
     }
 }

DistTestCore/Codex/CodexAccess.cs

@@ -19,7 +19,30 @@ namespace DistTestCore.Codex
 
         public CodexDebugResponse GetDebugInfo()
         {
-            return Http().HttpGetJson<CodexDebugResponse>("debug/info");
+            return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugResponse>("debug/info");
+        }
+
+        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        {
+            return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
+        }
+
+        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
+        {
+            var http = Http(timeout);
+            var str = http.HttpGetString($"debug/peer/{peerId}");
+
+            if (str.ToLowerInvariant() == "unable to find peer!")
+            {
+                return new CodexDebugPeerResponse
+                {
+                    IsPeerFound = false
+                };
+            }
+
+            var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
+            result.IsPeerFound = true;
+            return result;
         }
 
         public string UploadFile(FileStream fileStream)
@@ -42,6 +65,11 @@ namespace DistTestCore.Codex
             return Http().HttpPostJson($"storage/request/{contentId}", request);
         }
 
+        public string ConnectToPeer(string peerId, string peerMultiAddress)
+        {
+            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
+        }
+
         public void EnsureOnline()
         {
             try
@@ -51,7 +79,8 @@ namespace DistTestCore.Codex
 
                 var nodePeerId = debugInfo.id;
                 var nodeName = Container.Name;
-                log.AddStringReplace(nodePeerId, $"___{nodeName}___");
+                log.AddStringReplace(nodePeerId, nodeName);
+                log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
             }
             catch (Exception e)
             {
@@ -60,16 +89,11 @@ namespace DistTestCore.Codex
             }
         }
 
-        private Http Http()
+        private Http Http(TimeSpan? timeoutOverride = null)
         {
-            var ip = Container.Pod.Cluster.IP;
+            var ip = Container.Pod.Cluster.HostAddress;
             var port = Container.ServicePorts[0].Number;
-            return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1");
+            return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1", timeoutOverride);
-        }
-
-        public string ConnectToPeer(string peerId, string peerMultiAddress)
-        {
-            return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
         }
     }
 
@@ -82,6 +106,22 @@ namespace DistTestCore.Codex
         public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
         public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
         public CodexDebugVersionResponse codex { get; set; } = new();
+        public CodexDebugTableResponse table { get; set; } = new();
+    }
+
+    public class CodexDebugTableResponse
+    {
+        public CodexDebugTableNodeResponse localNode { get; set; } = new();
+        public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
+    }
+
+    public class CodexDebugTableNodeResponse
+    {
+        public string nodeId { get; set; } = string.Empty;
+        public string peerId { get; set; } = string.Empty;
+        public string record { get; set; } = string.Empty;
+        public string address { get; set; } = string.Empty;
+        public bool seen { get; set; }
     }
 
     public class EnginePeerResponse
@@ -110,6 +150,20 @@ namespace DistTestCore.Codex
         public string revision { get; set; } = string.Empty;
     }
 
+    public class CodexDebugPeerResponse
+    {
+        public bool IsPeerFound { get; set; }
+
+        public string peerId { get; set; } = string.Empty;
+        public long seqNo { get; set; }
+        public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
+    }
+
+    public class CodexDebugPeerAddressResponse
+    {
+        public string address { get; set; } = string.Empty;
+    }
+
     public class CodexSalesAvailabilityRequest
     {
         public string size { get; set; } = string.Empty;

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -13,6 +13,7 @@ namespace DistTestCore.Codex
             public const string DockerImage = "thatbenbierens/codexlocal:latest";
         #endif
         public const string MetricsPortTag = "metrics_port";
+        public const string DiscoveryPortTag = "discovery-port";
 
         protected override string Image => DockerImage;
 
@@ -22,7 +23,8 @@ namespace DistTestCore.Codex
 
             AddExposedPortAndVar("API_PORT");
             AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}");
-            AddInternalPortAndVar("DISC_PORT");
+            AddInternalPortAndVar("DISC_PORT", DiscoveryPortTag);
+            AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
 
             var listenPort = AddInternalPort();
             AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
@@ -31,11 +33,6 @@ namespace DistTestCore.Codex
             {
                 AddEnvVar("BOOTSTRAP_SPR", config.BootstrapSpr);
             }
-
-            if (config.LogLevel != null)
-            {
-                AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
-            }
             if (config.StorageQuota != null)
             {
                 AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);

DistTestCore/Codex/CodexStartupConfig.cs

@@ -5,9 +5,14 @@ namespace DistTestCore.Codex
 {
     public class CodexStartupConfig
     {
+        public CodexStartupConfig(CodexLogLevel logLevel)
+        {
+            LogLevel = logLevel;
+        }
+
         public string? NameOverride { get; set; }
         public Location Location { get; set; }
-        public CodexLogLevel? LogLevel { get; set; }
+        public CodexLogLevel LogLevel { get; }
         public ByteSize? StorageQuota { get; set; }
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }

DistTestCore/CodexSetup.cs

@@ -8,7 +8,6 @@ namespace DistTestCore
     {
         ICodexSetup WithName(string name);
         ICodexSetup At(Location location);
-        ICodexSetup WithLogLevel(CodexLogLevel level);
         ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
         ICodexSetup WithStorageQuota(ByteSize storageQuota);
         ICodexSetup EnableMetrics();
@@ -20,7 +19,8 @@ namespace DistTestCore
     {
         public int NumberOfNodes { get; }
 
-        public CodexSetup(int numberOfNodes)
+        public CodexSetup(int numberOfNodes, CodexLogLevel logLevel)
+            : base(logLevel)
         {
             NumberOfNodes = numberOfNodes;
         }
@@ -43,12 +43,6 @@ namespace DistTestCore
             return this;
         }
 
-        public ICodexSetup WithLogLevel(CodexLogLevel level)
-        {
-            LogLevel = level;
-            return this;
-        }
-
         public ICodexSetup WithStorageQuota(ByteSize storageQuota)
         {
             StorageQuota = storageQuota;
@@ -80,7 +74,7 @@ namespace DistTestCore
 
         private IEnumerable<string> DescribeArgs()
         {
-            if (LogLevel != null) yield return $"LogLevel={LogLevel}";
+            yield return $"LogLevel={LogLevel}";
             if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
             if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
         }

DistTestCore/CodexStarter.cs

@@ -1,6 +1,7 @@
 using DistTestCore.Codex;
 using DistTestCore.Marketplace;
 using KubernetesWorkflow;
+using Logging;
 
 namespace DistTestCore
 {
@@ -74,7 +75,7 @@ namespace DistTestCore
         {
             var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
             RunningGroups.Add(group);
-            group.EnsureOnline();
+            Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
             return group;
         }
 

DistTestCore/Configuration.cs

@@ -1,4 +1,5 @@
-using KubernetesWorkflow;
+using DistTestCore.Codex;
+using KubernetesWorkflow;
 
 namespace DistTestCore
 {
@@ -28,5 +29,10 @@ namespace DistTestCore
         {
             return "TestDataFiles";
         }
+
+        public CodexLogLevel GetCodexLogLevel()
+        {
+            return CodexLogLevel.Trace;
+        }
     }
 }

DistTestCore/DistTest.cs

@@ -1,4 +1,5 @@
 using DistTestCore.Codex;
+using DistTestCore.Helpers;
 using DistTestCore.Logs;
 using DistTestCore.Marketplace;
 using DistTestCore.Metrics;
@@ -25,8 +26,14 @@ namespace DistTestCore
             testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
 
             fixtureLog = new FixtureLog(configuration.GetLogConfig());
+
+            PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
+            PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
         }
 
+        public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; }
+        public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; }
+
         [OneTimeSetUp]
         public void GlobalSetup()
         {
@@ -85,6 +92,17 @@ namespace DistTestCore
             return Get().FileManager.GenerateTestFile(size);
         }
 
+        /// <summary>
+        /// Any test files generated in 'action' will be deleted after it returns.
+        /// This helps prevent large tests from filling up discs.
+        /// </summary>
+        public void ScopedTestFiles(Action action)
+        {
+            Get().FileManager.PushFileSet();
+            action();
+            Get().FileManager.PopFileSet();
+        }
+
         public IOnlineCodexNode SetupCodexBootstrapNode()
         {
             return SetupCodexBootstrapNode(s => { });
@@ -116,7 +134,7 @@ namespace DistTestCore
 
         public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
         {
-            var codexSetup = new CodexSetup(numberOfNodes);
+            var codexSetup = CreateCodexSetup(numberOfNodes);
 
             setup(codexSetup);
 
@@ -128,16 +146,31 @@ namespace DistTestCore
             return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
         }
 
-        protected void Log(string msg)
+        public IEnumerable<IOnlineCodexNode> GetAllOnlineCodexNodes()
         {
-            TestContext.Progress.WriteLine(msg);
+            return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
-            Get().Log.Log(msg);
         }
 
-        protected void Debug(string msg)
+        public BaseLog GetTestLog()
+        {
+            return Get().Log;
+        }
+
+        public void Log(string msg)
         {
             TestContext.Progress.WriteLine(msg);
-            Get().Log.Debug(msg);
+            GetTestLog().Log(msg);
+        }
+
+        public void Debug(string msg)
+        {
+            TestContext.Progress.WriteLine(msg);
+            GetTestLog().Debug(msg);
+        }
+
+        protected CodexSetup CreateCodexSetup(int numberOfNodes)
+        {
+            return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
         }
 
         private TestLifecycle Get()

DistTestCore/FileManager.cs

@@ -9,6 +9,8 @@ namespace DistTestCore
         TestFile CreateEmptyTestFile();
         TestFile GenerateTestFile(ByteSize size);
         void DeleteAllTestFiles();
+        void PushFileSet();
+        void PopFileSet();
     }
 
     public class FileManager : IFileManager
@@ -18,6 +20,7 @@ namespace DistTestCore
         private readonly Random random = new Random();
         private readonly TestLog log;
         private readonly string folder;
+        private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();
 
         public FileManager(TestLog log, Configuration configuration)
         {
@@ -31,6 +34,7 @@ namespace DistTestCore
         {
             var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin"));
             File.Create(result.Filename).Close();
+            if (fileSetStack.Any()) fileSetStack.Last().Add(result);
             return result;
         }
 
@@ -47,6 +51,27 @@ namespace DistTestCore
             DeleteDirectory();
         }
 
+        public void PushFileSet()
+        {
+            fileSetStack.Add(new List<TestFile>());
+        }
+
+        public void PopFileSet()
+        {
+            if (!fileSetStack.Any()) return;
+            var pop = fileSetStack.Last();
+            fileSetStack.Remove(pop);
+
+            foreach (var file in pop)
+            {
+                try
+                {
+                    File.Delete(file.Filename);
+                }
+                catch { }
+            }
+        }
+
         private void GenerateFileBytes(TestFile result, ByteSize size)
         {
             long bytesLeft = size.SizeInBytes;

DistTestCore/Helpers/PeerConnectionTestHelpers.cs

@@ -0,0 +1,239 @@
+using DistTestCore.Codex;
+using NUnit.Framework;
+using Utils;
+
+namespace DistTestCore.Helpers
+{
+    public class PeerConnectionTestHelpers
+    {
+        private readonly Random random = new Random();
+        private readonly DistTest test;
+
+        public PeerConnectionTestHelpers(DistTest test)
+        {
+            this.test = test;
+        }
+
+        public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullyConnected(nodes.ToArray());
+        }
+
+        public void AssertFullyConnected(params IOnlineCodexNode[] nodes)
+        {
+            test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            var entries = CreateEntries(nodes);
+            var pairs = CreatePairs(entries);
+
+            RetryWhilePairs(pairs, () =>
+            {
+                CheckAndRemoveSuccessful(pairs);
+            });
+
+            if (pairs.Any())
+            {
+                test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
+                Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
+                test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
+            }
+            else
+            {
+                test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
+            }
+        }
+
+        private static void RetryWhilePairs(List<Pair> pairs, Action action)
+        {
+            var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(10);
+            while (pairs.Any() && timeout > DateTime.UtcNow)
+            {
+                action();
+
+                if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(5));
+            }
+        }
+
+        private void CheckAndRemoveSuccessful(List<Pair> pairs)
+        {
+            var checkTasks = pairs.Select(p => Task.Run(() =>
+            {
+                ApplyRandomDelay();
+                p.Check();
+            })).ToArray();
+
+            Task.WaitAll(checkTasks);
+
+            foreach (var pair in pairs.ToArray())
+            {
+                if (pair.Success)
+                {
+                    test.Log(pair.GetMessage());
+                    pairs.Remove(pair);
+                }
+            }
+        }
+
+        private static Entry[] CreateEntries(IOnlineCodexNode[] nodes)
+        {
+            var entries = nodes.Select(n => new Entry(n)).ToArray();
+            var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
+
+            if (incorrectDiscoveryEndpoints.Any())
+            {
+                Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
+                    string.Join(Environment.NewLine, incorrectDiscoveryEndpoints));
+            }
+
+            return entries;
+        }
+
+        private static List<Pair> CreatePairs(Entry[] entries)
+        {
+            return CreatePairsIterator(entries).ToList();
+        }
+
+        private static IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
+        {
+            for (var x = 0; x < entries.Length; x++)
+            {
+                for (var y = x + 1; y < entries.Length; y++)
+                {
+                    yield return new Pair(entries[x], entries[y]);
+                }
+            }
+        }
+
+        private void ApplyRandomDelay()
+        {
+            // Calling all the nodes all at the same time is not exactly nice.
+            Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000)));
+        }
+
+        public class Entry
+        {
+            public Entry(IOnlineCodexNode node)
+            {
+                Node = node;
+                Response = node.GetDebugInfo();
+            }
+
+            public IOnlineCodexNode Node { get; }
+            public CodexDebugResponse Response { get; }
+
+            public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
+            {
+                foreach (var peer in Response.table.nodes)
+                {
+                    var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
+                    if (expected != peer.address)
+                    {
+                        yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'";
+                    }
+                }
+            }
+
+            private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
+            {
+                var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
+                if (peer == null) return $"peerId: {node.peerId} is not known.";
+
+                var n = (OnlineCodexNode)peer.Node;
+                var ip = n.CodexAccess.Container.Pod.Ip;
+                var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
+                return $"{ip}:{discPort.Number}";
+            }
+        }
+
+        public enum PeerConnectionState
+        {
+            Unknown,
+            Connection,
+            NoConnection,
+        }
+
+        public class Pair
+        {
+            private readonly TimeSpan timeout = TimeSpan.FromSeconds(60);
+            private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
+            private TimeSpan bToATime = TimeSpan.FromSeconds(0);
+
+            public Pair(Entry a, Entry b)
+            {
+                A = a;
+                B = b;
+            }
+
+            public Entry A { get; }
+            public Entry B { get; }
+            public PeerConnectionState AKnowsB { get; private set; }
+            public PeerConnectionState BKnowsA { get; private set; }
+            public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
+
+            public void Check()
+            {
+                aToBTime = Measure(() => AKnowsB = Knows(A, B));
+                bToATime = Measure(() => BKnowsA = Knows(B, A));
+            }
+
+            public string GetMessage()
+            {
+                return GetResultMessage() + GetTimePostfix();
+            }
+
+            private string GetResultMessage()
+            {
+                var aName = A.Response.id;
+                var bName = B.Response.id;
+
+                if (Success)
+                {
+                    return $"{aName} and {bName} know each other.";
+                }
+
+                return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}";
+            }
+
+            private string GetTimePostfix()
+            {
+                var aName = A.Response.id;
+                var bName = B.Response.id;
+
+                return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
+            }
+
+            private static TimeSpan Measure(Action action)
+            {
+                var start = DateTime.UtcNow;
+                action();
+                return DateTime.UtcNow - start;
+            }
+
+            private PeerConnectionState Knows(Entry a, Entry b)
+            {
+                lock (a)
+                {
+                    var peerId = b.Response.id;
+
+                    try
+                    {
+                        var response = a.Node.GetDebugPeer(peerId, timeout);
+                        if (!response.IsPeerFound)
+                        {
+                            return PeerConnectionState.NoConnection;
+                        }
+                        if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
+                        {
+                            return PeerConnectionState.Connection;
+                        }
+                    }
+                    catch
+                    {
+                    }
+
+                    // Didn't get a conclusive answer. Try again later.
+                    return PeerConnectionState.Unknown;
+                }
+            }
+        }
+    }
+}

DistTestCore/Helpers/PeerDownloadTestHelpers.cs

@@ -0,0 +1,52 @@
+namespace DistTestCore.Helpers
+{
+    public class PeerDownloadTestHelpers
+    {
+        private readonly DistTest test;
+
+        public PeerDownloadTestHelpers(DistTest test)
+        {
+            this.test = test;
+        }
+
+        public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullDownloadInterconnectivity(nodes, 1.MB());
+        }
+
+        public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
+        {
+            test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
+            foreach (var node in nodes)
+            {
+                var uploader = node;
+                var downloaders = nodes.Where(n => n != uploader).ToArray();
+
+                test.ScopedTestFiles(() =>
+                {
+                    PerformTest(uploader, downloaders, testFileSize);
+                });
+            }
+            
+            test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
+        }
+
+        private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
+        {
+            // 1 test file per downloader.
+            var files = downloaders.Select(d => test.GenerateTestFile(testFileSize)).ToArray();
+
+            // Upload all the test files to the uploader.
+            var contentIds = files.Select(uploader.UploadFile).ToArray();
+
+            // Each downloader should retrieve its own test file.
+            for (var i = 0; i < downloaders.Length; i++)
+            {
+                var expectedFile = files[i];
+                var downloadedFile = downloaders[i].DownloadContent(contentIds[i]);
+
+                expectedFile.AssertIsEqual(downloadedFile);
+            }
+        }
+    }
+}

DistTestCore/Http.cs

@@ -1,6 +1,5 @@
 using Logging;
 using Newtonsoft.Json;
-using NUnit.Framework;
 using System.Net.Http.Headers;
 using System.Net.Http.Json;
 using Utils;
@@ -14,15 +13,16 @@ namespace DistTestCore
         private readonly string ip;
         private readonly int port;
         private readonly string baseUrl;
+        private readonly TimeSpan? timeoutOverride;
 
-        public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl)
+        public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl, TimeSpan? timeoutOverride = null)
         {
             this.log = log;
             this.timeSet = timeSet;
             this.ip = ip;
             this.port = port;
             this.baseUrl = baseUrl;
-
+            this.timeoutOverride = timeoutOverride;
             if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
             if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
         }
@@ -38,7 +38,7 @@ namespace DistTestCore
                 var str = Time.Wait(result.Content.ReadAsStringAsync());
                 Log(url, str);
                 return str; ;
-            });
+            }, $"HTTP-GET:{route}");
         }
 
         public T HttpGetJson<T>(string route)
@@ -62,10 +62,10 @@ namespace DistTestCore
                 using var content = JsonContent.Create(body);
                 Log(url, JsonConvert.SerializeObject(body));
                 var result = Time.Wait(client.PostAsync(url, content));
-                var str= Time.Wait(result.Content.ReadAsStringAsync());
+                var str = Time.Wait(result.Content.ReadAsStringAsync());
                 Log(url, str);
                 return str;
-            });
+            }, $"HTTP-POST-JSON: {route}");
         }
 
         public string HttpPostStream(string route, Stream stream)
@@ -81,7 +81,7 @@ namespace DistTestCore
                 var str =Time.Wait(response.Content.ReadAsStringAsync());
                 Log(url, str);
                 return str;
-            });
+            }, $"HTTP-POST-STREAM: {route}");
         }
 
         public Stream HttpGetStream(string route)
@@ -92,7 +92,20 @@ namespace DistTestCore
                 var url = GetUrl() + route;
                 Log(url, "~ STREAM ~");
                 return Time.Wait(client.GetStreamAsync(url));
-            });
+            }, $"HTTP-GET-STREAM: {route}");
+        }
+
+        public T TryJsonDeserialize<T>(string json)
+        {
+            try
+            {
+                return JsonConvert.DeserializeObject<T>(json)!;
+            }
+            catch (Exception exception)
+            {
+                var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}";
+                throw new InvalidOperationException(msg, exception);
+            }
         }
 
         private string GetUrl()
@@ -105,47 +118,24 @@ namespace DistTestCore
             log.Debug($"({url}) = '{message}'", 3);
         }
 
-        private T Retry<T>(Func<T> operation)
+        private T Retry<T>(Func<T> operation, string description)
         {
-            var retryCounter = 0;
+            return Time.Retry(operation, timeSet.HttpCallRetryTimeout(), timeSet.HttpCallRetryDelay(), description);
-
-            while (true)
-            {
-                try
-                {
-                    return operation();
-                }
-                catch (Exception exception)
-                {
-                    timeSet.HttpCallRetryDelay();
-                    retryCounter++;
-                    if (retryCounter > timeSet.HttpCallRetryCount())
-                    {
-                        Assert.Fail(exception.ToString());
-                        throw;
-                    }
-                }
-            }
-        }
-
-        private static T TryJsonDeserialize<T>(string json)
-        {
-            try
-            {
-                return JsonConvert.DeserializeObject<T>(json)!;
-            }
-            catch (Exception exception)
-            {
-                var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}";
-                Assert.Fail(msg);
-                throw new InvalidOperationException(msg, exception);
-            }
         }
 
         private HttpClient GetClient()
+        {
+            if (timeoutOverride.HasValue)
+            {
+                return GetClient(timeoutOverride.Value);
+            }
+            return GetClient(timeSet.HttpCallTimeout());
+        }
+
+        private HttpClient GetClient(TimeSpan timeout)
         {
             var client = new HttpClient();
-            client.Timeout = timeSet.HttpCallTimeout();
+            client.Timeout = timeout;
             return client;
         }
     }

DistTestCore/Marketplace/ContainerInfoExtractor.cs

@@ -56,30 +56,6 @@ namespace DistTestCore.Marketplace
             return marketplaceAbi;
         }
 
-        private string Retry(Func<string> fetch)
-        {
-            var result = string.Empty;
-            Time.WaitUntil(() =>
-            {
-                result = Catch(fetch);
-                return !string.IsNullOrEmpty(result);
-            }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
-
-            return result;
-        }
-
-        private string Catch(Func<string> fetch)
-        {
-            try
-            {
-                return fetch();
-            }
-            catch
-            {
-                return string.Empty;
-            }
-        }
-
         private string FetchAccountsCsv()
         {
             return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename);
@@ -116,6 +92,11 @@ namespace DistTestCore.Marketplace
             var privateKey = tokens[1];
             return new GethAccount(account, privateKey);
         }
+
+        private static string Retry(Func<string> fetch)
+        {
+            return Time.Retry(fetch, nameof(ContainerInfoExtractor));
+        }
     }
 
     public class PubKeyFinder : LogHandler, ILogHandler

DistTestCore/Marketplace/GethBootstrapNodeInfo.cs

@@ -23,7 +23,7 @@ namespace DistTestCore.Marketplace
 
         public NethereumInteraction StartInteraction(BaseLog log)
         {
-            var ip = RunningContainers.RunningPod.Cluster.IP;
+            var ip = RunningContainers.RunningPod.Cluster.HostAddress;
             var port = RunningContainers.Containers[0].ServicePorts[0].Number;
             var account = Account;
 

DistTestCore/Marketplace/GethCompanionNodeInfo.cs

@@ -17,7 +17,7 @@ namespace DistTestCore.Marketplace
         
         public NethereumInteraction StartInteraction(BaseLog log, GethAccount account)
         {
-            var ip = RunningContainer.Pod.Cluster.IP;
+            var ip = RunningContainer.Pod.Cluster.HostAddress;
             var port = RunningContainer.ServicePorts[0].Number;
             var privateKey = account.PrivateKey;
 

DistTestCore/Metrics/MetricsQuery.cs

@@ -16,7 +16,7 @@ namespace DistTestCore.Metrics
             http = new Http(
                 log,
                 timeSet,
-                runningContainers.RunningPod.Cluster.IP,
+                runningContainers.RunningPod.Cluster.HostAddress,
                 runningContainers.Containers[0].ServicePorts[0].Number,
                 "api/v1");
         }

DistTestCore/OnlineCodexNode.cs

@@ -10,6 +10,8 @@ namespace DistTestCore
     {
         string GetName();
         CodexDebugResponse GetDebugInfo();
+        CodexDebugPeerResponse GetDebugPeer(string peerId);
+        CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout);
         ContentId UploadFile(TestFile file);
         TestFile? DownloadContent(ContentId contentId);
         void ConnectToPeer(IOnlineCodexNode node);
@@ -47,10 +49,21 @@ namespace DistTestCore
         public CodexDebugResponse GetDebugInfo()
         {
             var debugInfo = CodexAccess.GetDebugInfo();
-            Log($"Got DebugInfo with id: '{debugInfo.id}'.");
+            var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
+            Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
             return debugInfo;
         }
 
+        public CodexDebugPeerResponse GetDebugPeer(string peerId)
+        {
+            return CodexAccess.GetDebugPeer(peerId);
+        }
+
+        public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
+        {
+            return CodexAccess.GetDebugPeer(peerId, timeout);
+        }
+
         public ContentId UploadFile(TestFile file)
         {
             Log($"Uploading file of size {file.GetFileSize()}...");

DistTestCore/Timing.cs

@@ -1,5 +1,4 @@
 using NUnit.Framework;
-using Utils;
 
 namespace DistTestCore
 {
@@ -11,8 +10,8 @@ namespace DistTestCore
     public interface ITimeSet
     {
         TimeSpan HttpCallTimeout();
-        int HttpCallRetryCount();
+        TimeSpan HttpCallRetryTimeout();
-        void HttpCallRetryDelay();
+        TimeSpan HttpCallRetryDelay();
         TimeSpan WaitForK8sServiceDelay();
         TimeSpan K8sOperationTimeout();
         TimeSpan WaitForMetricTimeout();
@@ -25,14 +24,14 @@ namespace DistTestCore
             return TimeSpan.FromSeconds(10);
         }
 
-        public int HttpCallRetryCount()
+        public TimeSpan HttpCallRetryTimeout()
         {
-            return 5;
+            return TimeSpan.FromMinutes(1);
         }
 
-        public void HttpCallRetryDelay()
+        public TimeSpan HttpCallRetryDelay()
         {
-            Time.Sleep(TimeSpan.FromSeconds(3));
+            return TimeSpan.FromSeconds(3);
         }
 
         public TimeSpan WaitForK8sServiceDelay()
@@ -58,14 +57,14 @@ namespace DistTestCore
             return TimeSpan.FromHours(2);
         }
 
-        public int HttpCallRetryCount()
+        public TimeSpan HttpCallRetryTimeout()
         {
-            return 2;
+            return TimeSpan.FromHours(5);
         }
 
-        public void HttpCallRetryDelay()
+        public TimeSpan HttpCallRetryDelay()
         {
-            Time.Sleep(TimeSpan.FromMinutes(5));
+            return TimeSpan.FromMinutes(5);
         }
 
         public TimeSpan WaitForK8sServiceDelay()

KubernetesWorkflow/K8sCluster.cs

@@ -10,12 +10,12 @@ namespace KubernetesWorkflow
         }
        
         public Configuration Configuration { get; }
-        public string IP { get; private set; } = string.Empty;
+        public string HostAddress { get; private set; } = string.Empty;
 
         public KubernetesClientConfiguration GetK8sClientConfig()
         {
             var config = GetConfig();
-            UpdateIp(config);
+            UpdateHostAddress(config);
             return config;
         }
 
@@ -47,10 +47,17 @@ namespace KubernetesWorkflow
             }
         }
 
-        private void UpdateIp(KubernetesClientConfiguration config)
+        private void UpdateHostAddress(KubernetesClientConfiguration config)
         {
             var host = config.Host.Replace("https://", "");
-            IP = host.Substring(0, host.IndexOf(':'));
+            if (host.Contains(":"))
+            {
+                HostAddress = host.Substring(0, host.IndexOf(':'));
+            }
+            else
+            {
+                HostAddress = config.Host;
+            }
         }
     }
 }

LongTests/BasicTests/LargeFileTests.cs

@@ -1,5 +1,4 @@
 using DistTestCore;
-using DistTestCore.Codex;
 using NUnit.Framework;
 
 namespace TestsLong.BasicTests
@@ -11,7 +10,6 @@ namespace TestsLong.BasicTests
         public void OneClientLargeFileTest()
         {
             var primary = SetupCodexNode(s => s
-                                .WithLogLevel(CodexLogLevel.Warn)
                                 .WithStorageQuota(20.GB()));
 
             var testFile = GenerateTestFile(10.GB());

LongTests/BasicTests/TestInfraTests.cs

@@ -1,5 +1,4 @@
 using DistTestCore;
-using DistTestCore.Codex;
 using NUnit.Framework;
 
 namespace TestsLong.BasicTests
@@ -32,7 +31,6 @@ namespace TestsLong.BasicTests
         public void DownloadConsistencyTest()
         {
             var primary = SetupCodexNode(s => s
-                            .WithLogLevel(CodexLogLevel.Trace)
                             .WithStorageQuota(2.MB()));
 
             var testFile = GenerateTestFile(1.MB());

Tests/BasicTests/ExampleTests.cs

@@ -11,7 +11,7 @@ namespace Tests.BasicTests
         [Test]
         public void CodexLogExample()
         {
-            var primary = SetupCodexNode(s => s.WithLogLevel(CodexLogLevel.Trace));
+            var primary = SetupCodexNode();
 
             primary.UploadFile(GenerateTestFile(5.MB()));
 
@@ -47,7 +47,6 @@ namespace Tests.BasicTests
             var buyerInitialBalance = 1000.TestTokens();
 
             var seller = SetupCodexNode(s => s
-                            .WithLogLevel(CodexLogLevel.Trace)
                             .WithStorageQuota(11.GB())
                             .EnableMarketplace(sellerInitialBalance));
 
@@ -61,7 +60,6 @@ namespace Tests.BasicTests
             var testFile = GenerateTestFile(10.MB());
 
             var buyer = SetupCodexNode(s => s
-                .WithLogLevel(CodexLogLevel.Trace)
                 .WithBootstrapNode(seller)
                 .EnableMarketplace(buyerInitialBalance));
             

Tests/BasicTests/PeerTests.cs

@@ -1,85 +0,0 @@
-using DistTestCore;
-using DistTestCore.Codex;
-using NUnit.Framework;
-
-namespace Tests.BasicTests
-{
-    [TestFixture]
-    public class PeerTests : DistTest
-    {
-        [Test]
-        public void TwoNodes()
-        {
-            var primary = SetupCodexBootstrapNode();
-            var secondary = SetupCodexNode(s => s.WithBootstrapNode(primary));
-
-            primary.ConnectToPeer(secondary); // TODO REMOVE THIS: This is required for the switchPeers to show up.
-
-            // This is required for the enginePeers to show up.
-            //var file = GenerateTestFile(10.MB());
-            //var contentId = primary.UploadFile(file);
-            //var file2 = secondary.DownloadContent(contentId);
-            //file.AssertIsEqual(file2);
-
-            AssertKnowEachother(primary, secondary);
-        }
-
-        [TestCase(2)]
-        [TestCase(3)]
-        [TestCase(10)]
-        public void VariableNodes(int number)
-        {
-            var bootstrap = SetupCodexBootstrapNode();
-            var nodes = SetupCodexNodes(number, s => s.WithBootstrapNode(bootstrap));
-
-            var file = GenerateTestFile(10.MB());
-            var contentId = nodes.First().UploadFile(file);
-            var file2 = nodes.Last().DownloadContent(contentId);
-            file.AssertIsEqual(file2);
-
-            // <TODO REMOVE THIS>
-            foreach (var node in nodes) bootstrap.ConnectToPeer(node);
-            for (var x = 0; x < number; x++)
-            {
-                for (var y = x + 1; y < number; y++)
-                {
-                    nodes[x].ConnectToPeer(nodes[y]);
-                }
-            }
-            // </TODO REMOVE THIS>
-
-            foreach (var node in nodes) AssertKnowEachother(node, bootstrap);
-
-            for (var x = 0; x < number; x++)
-            {
-                for (var y = x + 1; y < number; y++)
-                {
-                    AssertKnowEachother(nodes[x], nodes[y]);
-                }
-            }
-        }
-
-        private void AssertKnowEachother(IOnlineCodexNode a, IOnlineCodexNode b)
-        {
-            AssertKnowEachother(a.GetDebugInfo(), b.GetDebugInfo());
-        }
-
-        private void AssertKnowEachother(CodexDebugResponse a, CodexDebugResponse b)
-        {
-            AssertKnows(a, b);
-            AssertKnows(b, a);
-        }
-
-        private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b)
-        {
-            var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
-            var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId));
-
-            Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]");
-            Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
-
-            Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
-            Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found.");
-        }
-    }
-}

Tests/DurabilityTests/DurabilityTests.cs

@@ -31,10 +31,10 @@ namespace Tests.DurabilityTests
         [Test]
         public void DataRetentionTest()
         {
-            var bootstrapNode = SetupCodexBootstrapNode(s => s.WithLogLevel(CodexLogLevel.Trace));
+            var bootstrapNode = SetupCodexBootstrapNode();
 
-            var startGroup = SetupCodexNodes(2, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode));
+            var startGroup = SetupCodexNodes(2, s => s.WithBootstrapNode(bootstrapNode));
-            var finishGroup = SetupCodexNodes(10, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode));
+            var finishGroup = SetupCodexNodes(10, s => s.WithBootstrapNode(bootstrapNode));
 
             var file = GenerateTestFile(10.MB());
 

Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs

@@ -0,0 +1,62 @@
+using DistTestCore;
+using DistTestCore.Helpers;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.PeerDiscoveryTests
+{
+    [TestFixture]
+    public class LayeredDiscoveryTests : DistTest
+    {
+        [Test]
+        public void TwoLayersTest()
+        {
+            var root = SetupCodexNode();
+            var l1Source = SetupCodexNode(s => s.WithBootstrapNode(root));
+            var l1Node = SetupCodexNode(s => s.WithBootstrapNode(root));
+            var l2Target = SetupCodexNode(s => s.WithBootstrapNode(l1Node));
+
+            AssertAllNodesConnected();
+        }
+
+        [Test]
+        public void ThreeLayersTest()
+        {
+            var root = SetupCodexNode();
+            var l1Source = SetupCodexNode(s => s.WithBootstrapNode(root));
+            var l1Node = SetupCodexNode(s => s.WithBootstrapNode(root));
+            var l2Node = SetupCodexNode(s => s.WithBootstrapNode(l1Node));
+            var l3Target = SetupCodexNode(s => s.WithBootstrapNode(l2Node));
+
+            AssertAllNodesConnected();
+        }
+
+        [TestCase(3)]
+        [TestCase(5)]
+        [TestCase(10)]
+        [TestCase(20)]
+        [TestCase(50)]
+        public void NodeChainTest(int chainLength)
+        {
+            var node = SetupCodexNode();
+            for (var i = 1; i < chainLength; i++)
+            {
+                node = SetupCodexNode(s => s.WithBootstrapNode(node));
+            }
+
+            AssertAllNodesConnected();
+
+            for (int i = 0; i < 5; i++)
+            {
+                Time.Sleep(TimeSpan.FromSeconds(30));
+                AssertAllNodesConnected();
+            }
+        }
+
+        private void AssertAllNodesConnected()
+        {
+            PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
+            //PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes());
+        }
+    }
+}

Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs

@@ -0,0 +1,74 @@
+using DistTestCore;
+using DistTestCore.Codex;
+using DistTestCore.Helpers;
+using NUnit.Framework;
+using Utils;
+
+namespace Tests.PeerDiscoveryTests
+{
+    [TestFixture]
+    public class PeerDiscoveryTests : AutoBootstrapDistTest
+    {
+        [Test]
+        public void CanReportUnknownPeerId()
+        {
+            var unknownId = "16Uiu2HAkv2CHWpff3dj5iuVNERAp8AGKGNgpGjPexJZHSqUstfsK";
+            var node = SetupCodexNode();
+
+            var result = node.GetDebugPeer(unknownId);
+            Assert.That(result.IsPeerFound, Is.False);
+        }
+
+        [TestCase(2)]
+        [TestCase(3)]
+        [TestCase(10)]
+        public void VariableNodes(int number)
+        {
+            SetupCodexNodes(number);
+
+            AssertAllNodesConnected();
+        }
+
+        [TestCase(2)]
+        [TestCase(3)]
+        [TestCase(10)]
+        [TestCase(20)]
+        public void VariableNodesInPods(int number)
+        {
+            for (var i = 0; i < number; i++)
+            {
+                SetupCodexNode();
+            }
+
+            AssertAllNodesConnected();
+        }
+
+        [TestCase(3, 3)]
+        [TestCase(3, 5)]
+        [TestCase(3, 10)]
+        [TestCase(5, 10)]
+        [TestCase(3, 20)]
+        [TestCase(5, 20)]
+        public void StagedVariableNodes(int numberOfNodes, int numberOfStages)
+        {
+            for (var i = 0; i < numberOfStages; i++)
+            {
+                SetupCodexNodes(numberOfNodes);
+
+                AssertAllNodesConnected();
+            }
+
+            for (int i = 0; i < 5; i++)
+            {
+                Time.Sleep(TimeSpan.FromSeconds(30));
+                AssertAllNodesConnected();
+            }
+        }
+
+        private void AssertAllNodesConnected()
+        {
+            PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
+            //PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes());
+        }
+    }
+}

Utils/Time.cs

@@ -38,5 +38,74 @@
                 state = predicate();
             }
         }
+
+        public static void Retry(Action action, string description)
+        {
+            Retry(action, TimeSpan.FromMinutes(1), description);
+        }
+
+        public static T Retry<T>(Func<T> action, string description)
+        {
+            return Retry(action, TimeSpan.FromMinutes(1), description);
+        }
+
+        public static void Retry(Action action, TimeSpan timeout, string description)
+        {
+            Retry(action, timeout, TimeSpan.FromSeconds(1), description);
+        }
+
+        public static T Retry<T>(Func<T> action, TimeSpan timeout, string description)
+        {
+            return Retry(action, timeout, TimeSpan.FromSeconds(1), description);
+        }
+
+        public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime, string description)
+        {
+            var start = DateTime.UtcNow;
+            var exceptions = new List<Exception>();
+            while (true)
+            {
+                if (DateTime.UtcNow - start > timeout)
+                {
+                    throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
+                }
+
+                try
+                {
+                    action();
+                    return;
+                }
+                catch (Exception ex)
+                {
+                    exceptions.Add(ex);
+                }
+
+                Sleep(retryTime);
+            }
+        }
+
+        public static T Retry<T>(Func<T> action, TimeSpan timeout, TimeSpan retryTime, string description)
+        {
+            var start = DateTime.UtcNow;
+            var exceptions = new List<Exception>();
+            while (true)
+            {
+                if (DateTime.UtcNow - start > timeout)
+                {
+                    throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
+                }
+
+                try
+                {
+                    return action();
+                }
+                catch (Exception ex)
+                {
+                    exceptions.Add(ex);
+                }
+
+                Sleep(retryTime);
+            }
+        }
     }
 }