From ab7a334987d5859a5106c7931a1307dfda219a2f Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 29 May 2023 08:35:46 +0200 Subject: [PATCH] Adds assert that nodes have correct addresses in their table of peers. --- DistTestCore/Codex/CodexAccess.cs | 5 +- DistTestCore/Codex/CodexContainerRecipe.cs | 3 +- .../LayeredDiscoveryTests.cs | 8 ++++ .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 3 +- Tests/PeerDiscoveryTests/PeerTestHelpers.cs | 46 ++++++++++++++----- 5 files changed, 48 insertions(+), 17 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 292cf46f..b664bb92 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -66,8 +66,8 @@ namespace DistTestCore.Codex var nodePeerId = debugInfo.id; var nodeName = Container.Name; - log.AddStringReplace(nodePeerId, $"___{nodeName}___"); - log.AddStringReplace(debugInfo.table.localNode.nodeId, $"__{nodeName}__"); + log.AddStringReplace(nodePeerId, nodeName); + log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName); } catch (Exception e) { @@ -107,6 +107,7 @@ namespace DistTestCore.Codex public string nodeId { get; set; } = string.Empty; public string peerId { get; set; } = string.Empty; public string record { get; set; } = string.Empty; + public string address { get; set; } = string.Empty; public bool seen { get; set; } } diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index ff263a75..12023e8d 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -13,6 +13,7 @@ namespace DistTestCore.Codex public const string DockerImage = "thatbenbierens/codexlocal:latest"; #endif public const string MetricsPortTag = "metrics_port"; + public const string DiscoveryPortTag = "discovery-port"; protected override string Image => DockerImage; @@ -22,7 +23,7 @@ namespace DistTestCore.Codex AddExposedPortAndVar("API_PORT"); AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}"); - AddInternalPortAndVar("DISC_PORT"); + AddInternalPortAndVar("DISC_PORT", DiscoveryPortTag); var listenPort = AddInternalPort(); AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); diff --git a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index 7028b234..5eaf1b42 100644 --- a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace Tests.PeerDiscoveryTests { @@ -33,6 +34,7 @@ namespace Tests.PeerDiscoveryTests [TestCase(5)] [TestCase(10)] [TestCase(20)] + [TestCase(50)] public void NodeChainTest(int chainLength) { var node = SetupCodexNode(); @@ -42,6 +44,12 @@ namespace Tests.PeerDiscoveryTests } AssertAllNodesConnected(); + + for (int i = 0; i < 5; i++) + { + Time.Sleep(TimeSpan.FromSeconds(30)); + AssertAllNodesConnected(); + } } private void AssertAllNodesConnected() diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index ad9b5c9f..1df3e847 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -46,8 +46,7 @@ namespace Tests.PeerDiscoveryTests AssertAllNodesConnected(); } - // Retain for a while - for (int i = 0; i < 10; i++) + for (int i = 0; i < 5; i++) { Time.Sleep(TimeSpan.FromSeconds(30)); AssertAllNodesConnected(); diff --git a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs index d9b70a6f..f3f4f0f0 100644 --- a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs +++ b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs @@ -52,18 +52,22 @@ namespace Tests.PeerDiscoveryTests if (pair.Success) { pairs.Remove(pair); - if (log != null) log.Log(pair.GetMessage()); - } - else - { - pair.IncreaseTimeout(); } } } private static Entry[] CreateEntries(IOnlineCodexNode[] nodes) { - return nodes.Select(n => new Entry(n)).ToArray(); + var entries = nodes.Select(n => new Entry(n)).ToArray(); + var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray(); + + if (incorrectDiscoveryEndpoints.Any()) + { + Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " + + string.Join(Environment.NewLine, incorrectDiscoveryEndpoints)); + } + + return entries; } private static List CreatePairs(Entry[] entries) @@ -92,11 +96,34 @@ namespace Tests.PeerDiscoveryTests public IOnlineCodexNode Node { get ; } public CodexDebugResponse Response { get; } + + public IEnumerable GetInCorrectDiscoveryEndpoints(Entry[] allEntries) + { + foreach (var peer in Response.table.nodes) + { + var expected = GetExpectedDiscoveryEndpoint(allEntries, peer); + if (expected != peer.address) + { + yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'"; + } + } + } + + private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node) + { + var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId); + if (peer == null) return $"peerId: {node.peerId} is not known."; + + var n = (OnlineCodexNode)peer.Node; + var ip = n.CodexAccess.Container.Pod.Ip; + var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag); + return $"{ip}:{discPort.Number}"; + } } public class Pair { - private TimeSpan timeout = TimeSpan.FromSeconds(60); + private readonly TimeSpan timeout = TimeSpan.FromSeconds(60); private TimeSpan aToBTime = TimeSpan.FromSeconds(0); private TimeSpan bToATime = TimeSpan.FromSeconds(0); @@ -124,11 +151,6 @@ namespace Tests.PeerDiscoveryTests return GetResultMessage() + GetTimePostfix(); } - public void IncreaseTimeout() - { - //timeout *= 2; - } - private string GetResultMessage() { var aName = A.Response.id;