From 1d224cf2d32cc99c994d88ee449f05f334f4a661 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 10 May 2023 08:53:57 +0200 Subject: [PATCH 01/16] Sets up working peer-discovery tests --- DistTestCore/AutoBootstrapDistTest.cs | 8 ++ DistTestCore/Codex/CodexAccess.cs | 14 +++ Tests/BasicTests/PeerTests.cs | 85 --------------- .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 103 ++++++++++++++++++ 4 files changed, 125 insertions(+), 85 deletions(-) delete mode 100644 Tests/BasicTests/PeerTests.cs create mode 100644 Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs diff --git a/DistTestCore/AutoBootstrapDistTest.cs b/DistTestCore/AutoBootstrapDistTest.cs index 352e2ec..1ffce5a 100644 --- a/DistTestCore/AutoBootstrapDistTest.cs +++ b/DistTestCore/AutoBootstrapDistTest.cs @@ -18,6 +18,14 @@ return BringOnline(codexSetup); } + protected IOnlineCodexNode BootstrapNode + { + get + { + return EnsureBootstapNode(); + } + } + private IOnlineCodexNode EnsureBootstapNode() { if (bootstrapNode == null) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 8654878..2cee677 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -82,6 +82,20 @@ namespace DistTestCore.Codex public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty(); public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty(); public CodexDebugVersionResponse codex { get; set; } = new(); + public CodexDebugTableResponse table { get; set; } = new(); + } + + public class CodexDebugTableResponse + { + public CodexDebugTableNodeResponse localNode { get; set; } = new(); + public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty(); + } + + public class CodexDebugTableNodeResponse + { + public string nodeId { get; set; } = string.Empty; + public string record { get; set; } = string.Empty; + public bool seen { get; set; } } public class EnginePeerResponse diff --git a/Tests/BasicTests/PeerTests.cs b/Tests/BasicTests/PeerTests.cs deleted file mode 100644 index 09b41ac..0000000 --- a/Tests/BasicTests/PeerTests.cs +++ /dev/null @@ -1,85 +0,0 @@ -using DistTestCore; -using DistTestCore.Codex; -using NUnit.Framework; - -namespace Tests.BasicTests -{ - [TestFixture] - public class PeerTests : DistTest - { - [Test] - public void TwoNodes() - { - var primary = SetupCodexBootstrapNode(); - var secondary = SetupCodexNode(s => s.WithBootstrapNode(primary)); - - primary.ConnectToPeer(secondary); // TODO REMOVE THIS: This is required for the switchPeers to show up. - - // This is required for the enginePeers to show up. - //var file = GenerateTestFile(10.MB()); - //var contentId = primary.UploadFile(file); - //var file2 = secondary.DownloadContent(contentId); - //file.AssertIsEqual(file2); - - AssertKnowEachother(primary, secondary); - } - - [TestCase(2)] - [TestCase(3)] - [TestCase(10)] - public void VariableNodes(int number) - { - var bootstrap = SetupCodexBootstrapNode(); - var nodes = SetupCodexNodes(number, s => s.WithBootstrapNode(bootstrap)); - - var file = GenerateTestFile(10.MB()); - var contentId = nodes.First().UploadFile(file); - var file2 = nodes.Last().DownloadContent(contentId); - file.AssertIsEqual(file2); - - // - foreach (var node in nodes) bootstrap.ConnectToPeer(node); - for (var x = 0; x < number; x++) - { - for (var y = x + 1; y < number; y++) - { - nodes[x].ConnectToPeer(nodes[y]); - } - } - // - - foreach (var node in nodes) AssertKnowEachother(node, bootstrap); - - for (var x = 0; x < number; x++) - { - for (var y = x + 1; y < number; y++) - { - AssertKnowEachother(nodes[x], nodes[y]); - } - } - } - - private void AssertKnowEachother(IOnlineCodexNode a, IOnlineCodexNode b) - { - AssertKnowEachother(a.GetDebugInfo(), b.GetDebugInfo()); - } - - private void AssertKnowEachother(CodexDebugResponse a, CodexDebugResponse b) - { - AssertKnows(a, b); - AssertKnows(b, a); - } - - private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b) - { - var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); - var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); - - Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]"); - Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]"); - - Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); - Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); - } - } -} diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs new file mode 100644 index 0000000..23773f5 --- /dev/null +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -0,0 +1,103 @@ +using DistTestCore.Codex; +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace Tests.PeerDiscoveryTests +{ + public class PeerDiscoveryTests : AutoBootstrapDistTest + { + [Test] + public void TwoNodes() + { + var node = SetupCodexNode(); + + AssertKnowEachother(BootstrapNode, node); + } + + [TestCase(2)] + [TestCase(3)] + [TestCase(10)] + public void VariableNodes(int number) + { + var nodes = SetupCodexNodes(number); + + AssertFullyConnected(nodes); + } + + [TestCase(2)] + [TestCase(3)] + [TestCase(10)] + public void VariableNodesInPods(int number) + { + var bootstrap = SetupCodexBootstrapNode(); + + var nodes = new List(); + for (var i = 0; i < number; i++) + { + nodes.Add(SetupCodexNode(s => s.WithBootstrapNode(bootstrap))); + } + + AssertFullyConnected(nodes); + } + + private void AssertFullyConnected(IEnumerable nodes) + { + Retry(() => + { + var array = nodes.ToArray(); + + foreach (var node in array) AssertKnowEachother(node, BootstrapNode); + + for (var x = 0; x < array.Length; x++) + { + for (var y = x + 1; y < array.Length; y++) + { + AssertKnowEachother(array[x], array[y]); + } + } + }); + } + + private static void Retry(Action action) + { + try + { + action(); + return; + } + catch + { + Time.Sleep(TimeSpan.FromMinutes(1)); + } + + action(); + } + + private void AssertKnowEachother(IOnlineCodexNode a, IOnlineCodexNode b) + { + AssertKnowEachother(a.GetDebugInfo(), b.GetDebugInfo()); + } + + private void AssertKnowEachother(CodexDebugResponse a, CodexDebugResponse b) + { + AssertKnows(a, b); + AssertKnows(b, a); + } + + private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b) + { + //var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); + //var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); + var tableNodes = string.Join(",", a.table.nodes.Select(n => n.nodeId)); + + //Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]"); + //Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]"); + Debug($"{a.table.localNode.nodeId} is looking for {b.table.localNode.nodeId} in table-nodes [{tableNodes}]"); + + //Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); + //Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); + Assert.That(a.table.nodes.Any(n => n.nodeId == b.table.localNode.nodeId), $"{a.table.localNode.nodeId} was looking for '{b.table.localNode.nodeId}' in table-nodes [{tableNodes}] but it was not found."); + } + } +} From 2a55328d11d49e21295fababbe0b5eafd6112819 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 10 May 2023 09:09:31 +0200 Subject: [PATCH 02/16] Debugs AutoBootstrapDistTest --- DistTestCore/AutoBootstrapDistTest.cs | 25 ++++++------------- .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 1 + 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/DistTestCore/AutoBootstrapDistTest.cs b/DistTestCore/AutoBootstrapDistTest.cs index 1ffce5a..2ac0f8d 100644 --- a/DistTestCore/AutoBootstrapDistTest.cs +++ b/DistTestCore/AutoBootstrapDistTest.cs @@ -1,9 +1,9 @@ -namespace DistTestCore +using NUnit.Framework; + +namespace DistTestCore { public class AutoBootstrapDistTest : DistTest { - private IOnlineCodexNode? bootstrapNode; - public override IOnlineCodexNode SetupCodexBootstrapNode(Action setup) { throw new Exception("AutoBootstrapDistTest creates and attaches a single boostrap node for you. " + @@ -14,25 +14,16 @@ { var codexSetup = new CodexSetup(numberOfNodes); setup(codexSetup); - codexSetup.WithBootstrapNode(EnsureBootstapNode()); + codexSetup.WithBootstrapNode(BootstrapNode); return BringOnline(codexSetup); } - protected IOnlineCodexNode BootstrapNode + [SetUp] + public void SetUpBootstrapNode() { - get - { - return EnsureBootstapNode(); - } + BootstrapNode = BringOnline(new CodexSetup(1))[0]; } - private IOnlineCodexNode EnsureBootstapNode() - { - if (bootstrapNode == null) - { - bootstrapNode = base.SetupCodexBootstrapNode(s => { }); - } - return bootstrapNode; - } + protected IOnlineCodexNode BootstrapNode { get; private set; } = null!; } } diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 23773f5..2b3e4ff 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -5,6 +5,7 @@ using Utils; namespace Tests.PeerDiscoveryTests { + [TestFixture] public class PeerDiscoveryTests : AutoBootstrapDistTest { [Test] From d58cb38c79346d5f953c3dbb4cee79cab3ea0eae Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 10 May 2023 09:55:36 +0200 Subject: [PATCH 03/16] Cleanup duplicated retry logic. Adds layered and chain tests --- DistTestCore/DistTest.cs | 14 +++- DistTestCore/Http.cs | 20 +---- .../Marketplace/ContainerInfoExtractor.cs | 29 ++----- DistTestCore/Timing.cs | 21 +++-- .../LayeredDiscoveryTests.cs | 51 ++++++++++++ .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 77 +++---------------- Tests/PeerDiscoveryTests/PeerTestHelpers.cs | 57 ++++++++++++++ Utils/Time.cs | 69 +++++++++++++++++ 8 files changed, 214 insertions(+), 124 deletions(-) create mode 100644 Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs create mode 100644 Tests/PeerDiscoveryTests/PeerTestHelpers.cs diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index e5ee885..f0280b1 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -128,16 +128,26 @@ namespace DistTestCore return Get().CodexStarter.BringOnline((CodexSetup)codexSetup); } + public IEnumerable GetAllOnlineCodexNodes() + { + return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes); + } + + protected BaseLog GetTestLog() + { + return Get().Log; + } + protected void Log(string msg) { TestContext.Progress.WriteLine(msg); - Get().Log.Log(msg); + GetTestLog().Log(msg); } protected void Debug(string msg) { TestContext.Progress.WriteLine(msg); - Get().Log.Debug(msg); + GetTestLog().Debug(msg); } private TestLifecycle Get() diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 3dfee9d..bd9f34c 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -107,25 +107,7 @@ namespace DistTestCore private T Retry(Func operation) { - var retryCounter = 0; - - while (true) - { - try - { - return operation(); - } - catch (Exception exception) - { - timeSet.HttpCallRetryDelay(); - retryCounter++; - if (retryCounter > timeSet.HttpCallRetryCount()) - { - Assert.Fail(exception.ToString()); - throw; - } - } - } + return Time.Retry(operation, timeSet.HttpCallRetryTimeout(), timeSet.HttpCallRetryDelay()); } private static T TryJsonDeserialize(string json) diff --git a/DistTestCore/Marketplace/ContainerInfoExtractor.cs b/DistTestCore/Marketplace/ContainerInfoExtractor.cs index f99827b..5f6db11 100644 --- a/DistTestCore/Marketplace/ContainerInfoExtractor.cs +++ b/DistTestCore/Marketplace/ContainerInfoExtractor.cs @@ -56,30 +56,6 @@ namespace DistTestCore.Marketplace return marketplaceAbi; } - private string Retry(Func fetch) - { - var result = string.Empty; - Time.WaitUntil(() => - { - result = Catch(fetch); - return !string.IsNullOrEmpty(result); - }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3)); - - return result; - } - - private string Catch(Func fetch) - { - try - { - return fetch(); - } - catch - { - return string.Empty; - } - } - private string FetchAccountsCsv() { return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename); @@ -116,6 +92,11 @@ namespace DistTestCore.Marketplace var privateKey = tokens[1]; return new GethAccount(account, privateKey); } + + private static string Retry(Func fetch) + { + return Time.Retry(fetch); + } } public class PubKeyFinder : LogHandler, ILogHandler diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs index 9771767..e3f7414 100644 --- a/DistTestCore/Timing.cs +++ b/DistTestCore/Timing.cs @@ -1,5 +1,4 @@ using NUnit.Framework; -using Utils; namespace DistTestCore { @@ -11,8 +10,8 @@ namespace DistTestCore public interface ITimeSet { TimeSpan HttpCallTimeout(); - int HttpCallRetryCount(); - void HttpCallRetryDelay(); + TimeSpan HttpCallRetryTimeout(); + TimeSpan HttpCallRetryDelay(); TimeSpan WaitForK8sServiceDelay(); TimeSpan K8sOperationTimeout(); TimeSpan WaitForMetricTimeout(); @@ -25,14 +24,14 @@ namespace DistTestCore return TimeSpan.FromSeconds(10); } - public int HttpCallRetryCount() + public TimeSpan HttpCallRetryTimeout() { - return 5; + return TimeSpan.FromSeconds(10); } - public void HttpCallRetryDelay() + public TimeSpan HttpCallRetryDelay() { - Time.Sleep(TimeSpan.FromSeconds(3)); + return TimeSpan.FromSeconds(3); } public TimeSpan WaitForK8sServiceDelay() @@ -58,14 +57,14 @@ namespace DistTestCore return TimeSpan.FromHours(2); } - public int HttpCallRetryCount() + public TimeSpan HttpCallRetryTimeout() { - return 2; + return TimeSpan.FromHours(5); } - public void HttpCallRetryDelay() + public TimeSpan HttpCallRetryDelay() { - Time.Sleep(TimeSpan.FromMinutes(5)); + return TimeSpan.FromMinutes(5); } public TimeSpan WaitForK8sServiceDelay() diff --git a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs new file mode 100644 index 0000000..0be9db1 --- /dev/null +++ b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -0,0 +1,51 @@ +using DistTestCore; +using NUnit.Framework; + +namespace Tests.PeerDiscoveryTests +{ + [TestFixture] + public class LayeredDiscoveryTests : DistTest + { + [Test] + public void TwoLayersTest() + { + var root = SetupCodexNode(); + var l1Source = SetupCodexNode(s => s.WithBootstrapNode(root)); + var l1Node = SetupCodexNode(s => s.WithBootstrapNode(root)); + var l2Target = SetupCodexNode(s => s.WithBootstrapNode(l1Node)); + + AssertAllNodesConnected(); + } + + [Test] + public void ThreeLayersTest() + { + var root = SetupCodexNode(); + var l1Source = SetupCodexNode(s => s.WithBootstrapNode(root)); + var l1Node = SetupCodexNode(s => s.WithBootstrapNode(root)); + var l2Node = SetupCodexNode(s => s.WithBootstrapNode(l1Node)); + var l3Target = SetupCodexNode(s => s.WithBootstrapNode(l2Node)); + + AssertAllNodesConnected(); + } + + [TestCase(3)] + [TestCase(5)] + [TestCase(10)] + public void NodeChainTest(int chainLength) + { + var node = SetupCodexNode(); + for (var i = 1; i < chainLength; i++) + { + node = SetupCodexNode(s => s.WithBootstrapNode(node)); + } + + AssertAllNodesConnected(); + } + + private void AssertAllNodesConnected() + { + PeerTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes(), GetTestLog()); + } + } +} diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 2b3e4ff..9d5c735 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -1,7 +1,5 @@ -using DistTestCore.Codex; -using DistTestCore; +using DistTestCore; using NUnit.Framework; -using Utils; namespace Tests.PeerDiscoveryTests { @@ -11,9 +9,9 @@ namespace Tests.PeerDiscoveryTests [Test] public void TwoNodes() { - var node = SetupCodexNode(); + SetupCodexNode(); - AssertKnowEachother(BootstrapNode, node); + AssertAllNodesConnected(); } [TestCase(2)] @@ -21,9 +19,9 @@ namespace Tests.PeerDiscoveryTests [TestCase(10)] public void VariableNodes(int number) { - var nodes = SetupCodexNodes(number); + SetupCodexNodes(number); - AssertFullyConnected(nodes); + AssertAllNodesConnected(); } [TestCase(2)] @@ -31,74 +29,17 @@ namespace Tests.PeerDiscoveryTests [TestCase(10)] public void VariableNodesInPods(int number) { - var bootstrap = SetupCodexBootstrapNode(); - - var nodes = new List(); for (var i = 0; i < number; i++) { - nodes.Add(SetupCodexNode(s => s.WithBootstrapNode(bootstrap))); + SetupCodexNode(); } - AssertFullyConnected(nodes); + AssertAllNodesConnected(); } - private void AssertFullyConnected(IEnumerable nodes) + private void AssertAllNodesConnected() { - Retry(() => - { - var array = nodes.ToArray(); - - foreach (var node in array) AssertKnowEachother(node, BootstrapNode); - - for (var x = 0; x < array.Length; x++) - { - for (var y = x + 1; y < array.Length; y++) - { - AssertKnowEachother(array[x], array[y]); - } - } - }); - } - - private static void Retry(Action action) - { - try - { - action(); - return; - } - catch - { - Time.Sleep(TimeSpan.FromMinutes(1)); - } - - action(); - } - - private void AssertKnowEachother(IOnlineCodexNode a, IOnlineCodexNode b) - { - AssertKnowEachother(a.GetDebugInfo(), b.GetDebugInfo()); - } - - private void AssertKnowEachother(CodexDebugResponse a, CodexDebugResponse b) - { - AssertKnows(a, b); - AssertKnows(b, a); - } - - private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b) - { - //var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); - //var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); - var tableNodes = string.Join(",", a.table.nodes.Select(n => n.nodeId)); - - //Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]"); - //Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]"); - Debug($"{a.table.localNode.nodeId} is looking for {b.table.localNode.nodeId} in table-nodes [{tableNodes}]"); - - //Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); - //Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); - Assert.That(a.table.nodes.Any(n => n.nodeId == b.table.localNode.nodeId), $"{a.table.localNode.nodeId} was looking for '{b.table.localNode.nodeId}' in table-nodes [{tableNodes}] but it was not found."); + PeerTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes(), GetTestLog()); } } } diff --git a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs new file mode 100644 index 0000000..6b33ddd --- /dev/null +++ b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs @@ -0,0 +1,57 @@ +using DistTestCore.Codex; +using DistTestCore; +using NUnit.Framework; +using Utils; +using Logging; + +namespace Tests.PeerDiscoveryTests +{ + public static class PeerTestHelpers + { + public static void AssertFullyConnected(IEnumerable nodes, BaseLog? log = null) + { + AssertFullyConnected(log, nodes.ToArray()); + } + + public static void AssertFullyConnected(BaseLog? log = null, params IOnlineCodexNode[] nodes) + { + Time.Retry(() => + { + for (var x = 0; x < nodes.Length; x++) + { + for (var y = x + 1; y < nodes.Length; y++) + { + AssertKnowEachother(nodes[x], nodes[y], log); + } + } + }); + } + + private static void AssertKnowEachother(IOnlineCodexNode a, IOnlineCodexNode b, BaseLog? log) + { + AssertKnowEachother(a.GetDebugInfo(), b.GetDebugInfo(), log); + } + + private static void AssertKnowEachother(CodexDebugResponse a, CodexDebugResponse b, BaseLog? log) + { + AssertKnows(a, b, log); + AssertKnows(b, a, log); + } + + private static void AssertKnows(CodexDebugResponse a, CodexDebugResponse b, BaseLog? log) + { + //var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); + //var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); + var tableNodes = string.Join(",", a.table.nodes.Select(n => n.nodeId)); + + if (log != null) + { + log.Debug($"{a.table.localNode.nodeId} is looking for {b.table.localNode.nodeId} in table-nodes [{tableNodes}]"); + } + + //Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); + //Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); + Assert.That(a.table.nodes.Any(n => n.nodeId == b.table.localNode.nodeId), $"{a.table.localNode.nodeId} was looking for '{b.table.localNode.nodeId}' in table-nodes [{tableNodes}] but it was not found."); + } + } +} diff --git a/Utils/Time.cs b/Utils/Time.cs index 0f4f71b..2002b9c 100644 --- a/Utils/Time.cs +++ b/Utils/Time.cs @@ -38,5 +38,74 @@ state = predicate(); } } + + public static void Retry(Action action) + { + Retry(action, TimeSpan.FromMinutes(1)); + } + + public static T Retry(Func action) + { + return Retry(action, TimeSpan.FromMinutes(1)); + } + + public static void Retry(Action action, TimeSpan timeout) + { + Retry(action, timeout, TimeSpan.FromSeconds(1)); + } + + public static T Retry(Func action, TimeSpan timeout) + { + return Retry(action, timeout, TimeSpan.FromSeconds(1)); + } + + public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime) + { + var start = DateTime.UtcNow; + var exceptions = new List(); + while (true) + { + if (DateTime.UtcNow - start > timeout) + { + throw new TimeoutException("Retry timed out.", new AggregateException(exceptions)); + } + + try + { + action(); + return; + } + catch (Exception ex) + { + exceptions.Add(ex); + } + + Sleep(retryTime); + } + } + + public static T Retry(Func action, TimeSpan timeout, TimeSpan retryTime) + { + var start = DateTime.UtcNow; + var exceptions = new List(); + while (true) + { + if (DateTime.UtcNow - start > timeout) + { + throw new TimeoutException("Retry timed out.", new AggregateException(exceptions)); + } + + try + { + return action(); + } + catch (Exception ex) + { + exceptions.Add(ex); + } + + Sleep(retryTime); + } + } } } From e9679e18c0f0e462e97755fb575e4c3a56aac75d Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 10 May 2023 10:47:10 +0200 Subject: [PATCH 04/16] logs all checks before failing --- Tests/PeerDiscoveryTests/PeerTestHelpers.cs | 40 +++++++++++++++------ 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs index 6b33ddd..af2a3ac 100644 --- a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs +++ b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs @@ -17,41 +17,59 @@ namespace Tests.PeerDiscoveryTests { Time.Retry(() => { - for (var x = 0; x < nodes.Length; x++) + var infos = nodes.Select(n => FetchDebugInfo(n, log)).ToArray(); + + var failureMessags = new List(); + for (var x = 0; x < infos.Length; x++) { - for (var y = x + 1; y < nodes.Length; y++) + for (var y = x + 1; y < infos.Length; y++) { - AssertKnowEachother(nodes[x], nodes[y], log); + AssertKnowEachother(failureMessags, infos[x], infos[y], log); } } + + CollectionAssert.IsEmpty(failureMessags); }); } - private static void AssertKnowEachother(IOnlineCodexNode a, IOnlineCodexNode b, BaseLog? log) + private static CodexDebugResponse FetchDebugInfo(IOnlineCodexNode n, BaseLog? log) { - AssertKnowEachother(a.GetDebugInfo(), b.GetDebugInfo(), log); + var info = n.GetDebugInfo(); + + if (log != null) + { + log.AddStringReplace(info.table.localNode.nodeId, $"-<{n.GetName()}>-"); + } + + return info; } - private static void AssertKnowEachother(CodexDebugResponse a, CodexDebugResponse b, BaseLog? log) + private static void AssertKnowEachother(List failureMessags, CodexDebugResponse a, CodexDebugResponse b, BaseLog? log) { - AssertKnows(a, b, log); - AssertKnows(b, a, log); + AssertKnows(failureMessags, a, b, log); + AssertKnows(failureMessags, b, a, log); } - private static void AssertKnows(CodexDebugResponse a, CodexDebugResponse b, BaseLog? log) + private static void AssertKnows(List failureMessags, CodexDebugResponse a, CodexDebugResponse b, BaseLog? log) { //var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); //var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); var tableNodes = string.Join(",", a.table.nodes.Select(n => n.nodeId)); + var success = a.table.nodes.Any(n => n.nodeId == b.table.localNode.nodeId); + if (log != null) { - log.Debug($"{a.table.localNode.nodeId} is looking for {b.table.localNode.nodeId} in table-nodes [{tableNodes}]"); + var msg = success ? "PASS" : "FAIL"; + log.Log($"{msg} {a.table.localNode.nodeId} is looking for {b.table.localNode.nodeId} in table-nodes [{tableNodes}]"); } //Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); //Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); - Assert.That(a.table.nodes.Any(n => n.nodeId == b.table.localNode.nodeId), $"{a.table.localNode.nodeId} was looking for '{b.table.localNode.nodeId}' in table-nodes [{tableNodes}] but it was not found."); + if (!success) + { + failureMessags.Add($"{a.table.localNode.nodeId} was looking for '{b.table.localNode.nodeId}' in table-nodes [{tableNodes}] but it was not found."); + } } } } From 38d5b172f48a0dd38d46f528e27a9951ba5e483f Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 11 May 2023 12:44:53 +0200 Subject: [PATCH 05/16] successful peer discovery tests --- DistTestCore/Codex/CodexAccess.cs | 33 ++++++-- DistTestCore/CodexStarter.cs | 3 +- DistTestCore/Http.cs | 16 +++- DistTestCore/OnlineCodexNode.cs | 6 ++ DistTestCore/Timing.cs | 2 +- .../LayeredDiscoveryTests.cs | 1 + Tests/PeerDiscoveryTests/PeerTestHelpers.cs | 79 ++++++++++++------- 7 files changed, 97 insertions(+), 43 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 2cee677..00affc2 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -19,7 +19,12 @@ namespace DistTestCore.Codex public CodexDebugResponse GetDebugInfo() { - return Http().HttpGetJson("debug/info"); + return Http(TimeSpan.FromSeconds(2)).HttpGetJson("debug/info"); + } + + public CodexDebugPeerResponse GetDebugPeer(string peerId) + { + return Http().HttpGetJson($"debug/peer/{peerId}"); } public string UploadFile(FileStream fileStream) @@ -42,6 +47,11 @@ namespace DistTestCore.Codex return Http().HttpPostJson($"storage/request/{contentId}", request); } + public string ConnectToPeer(string peerId, string peerMultiAddress) + { + return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); + } + public void EnsureOnline() { try @@ -60,16 +70,11 @@ namespace DistTestCore.Codex } } - private Http Http() + private Http Http(TimeSpan? timeoutOverride = null) { var ip = Container.Pod.Cluster.IP; var port = Container.ServicePorts[0].Number; - return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1"); - } - - public string ConnectToPeer(string peerId, string peerMultiAddress) - { - return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); + return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1", timeoutOverride); } } @@ -124,6 +129,18 @@ namespace DistTestCore.Codex public string revision { get; set; } = string.Empty; } + public class CodexDebugPeerResponse + { + public string peerId { get; set; } = string.Empty; + public long seqNo { get; set; } + public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty(); + } + + public class CodexDebugPeerAddressResponse + { + public string address { get; set; } = string.Empty; + } + public class CodexSalesAvailabilityRequest { public string size { get; set; } = string.Empty; diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index e36ebca..1c49cad 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,6 +1,7 @@ using DistTestCore.Codex; using DistTestCore.Marketplace; using KubernetesWorkflow; +using Logging; namespace DistTestCore { @@ -74,7 +75,7 @@ namespace DistTestCore { var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); RunningGroups.Add(group); - group.EnsureOnline(); + Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true); return group; } diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index bd9f34c..a4feeb8 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -14,15 +14,16 @@ namespace DistTestCore private readonly string ip; private readonly int port; private readonly string baseUrl; + private readonly TimeSpan? timeoutOverride; - public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl) + public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl, TimeSpan? timeoutOverride = null) { this.log = log; this.timeSet = timeSet; this.ip = ip; this.port = port; this.baseUrl = baseUrl; - + this.timeoutOverride = timeoutOverride; if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; } @@ -125,9 +126,18 @@ namespace DistTestCore } private HttpClient GetClient() + { + if (timeoutOverride.HasValue) + { + return GetClient(timeoutOverride.Value); + } + return GetClient(timeSet.HttpCallTimeout()); + } + + private HttpClient GetClient(TimeSpan timeout) { var client = new HttpClient(); - client.Timeout = timeSet.HttpCallTimeout(); + client.Timeout = timeout; return client; } } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 7fb30e1..ccec842 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -10,6 +10,7 @@ namespace DistTestCore { string GetName(); CodexDebugResponse GetDebugInfo(); + CodexDebugPeerResponse GetDebugPeer(string peerId); ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); void ConnectToPeer(IOnlineCodexNode node); @@ -51,6 +52,11 @@ namespace DistTestCore return debugInfo; } + public CodexDebugPeerResponse GetDebugPeer(string peerId) + { + return CodexAccess.GetDebugPeer(peerId); + } + public ContentId UploadFile(TestFile file) { Log($"Uploading file of size {file.GetFileSize()}..."); diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs index e3f7414..cff8b09 100644 --- a/DistTestCore/Timing.cs +++ b/DistTestCore/Timing.cs @@ -26,7 +26,7 @@ namespace DistTestCore public TimeSpan HttpCallRetryTimeout() { - return TimeSpan.FromSeconds(10); + return TimeSpan.FromMinutes(1); } public TimeSpan HttpCallRetryDelay() diff --git a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index 0be9db1..7028b23 100644 --- a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -32,6 +32,7 @@ namespace Tests.PeerDiscoveryTests [TestCase(3)] [TestCase(5)] [TestCase(10)] + [TestCase(20)] public void NodeChainTest(int chainLength) { var node = SetupCodexNode(); diff --git a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs index af2a3ac..f607888 100644 --- a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs +++ b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs @@ -17,14 +17,14 @@ namespace Tests.PeerDiscoveryTests { Time.Retry(() => { - var infos = nodes.Select(n => FetchDebugInfo(n, log)).ToArray(); + var entries = nodes.Select(n => new Entry(n)).ToArray(); var failureMessags = new List(); - for (var x = 0; x < infos.Length; x++) + for (var x = 0; x < entries.Length; x++) { - for (var y = x + 1; y < infos.Length; y++) + for (var y = x + 1; y < entries.Length; y++) { - AssertKnowEachother(failureMessags, infos[x], infos[y], log); + AssertKnowEachother(failureMessags, entries[x], entries[y], log); } } @@ -32,44 +32,63 @@ namespace Tests.PeerDiscoveryTests }); } - private static CodexDebugResponse FetchDebugInfo(IOnlineCodexNode n, BaseLog? log) - { - var info = n.GetDebugInfo(); - - if (log != null) - { - log.AddStringReplace(info.table.localNode.nodeId, $"-<{n.GetName()}>-"); - } - - return info; - } - - private static void AssertKnowEachother(List failureMessags, CodexDebugResponse a, CodexDebugResponse b, BaseLog? log) + private static void AssertKnowEachother(List failureMessags, Entry a, Entry b, BaseLog? log) { AssertKnows(failureMessags, a, b, log); AssertKnows(failureMessags, b, a, log); } - private static void AssertKnows(List failureMessags, CodexDebugResponse a, CodexDebugResponse b, BaseLog? log) + private static void AssertKnows(List failureMessags, Entry a, Entry b, BaseLog? log) { - //var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); - //var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); - var tableNodes = string.Join(",", a.table.nodes.Select(n => n.nodeId)); + var peerId = b.Response.id; - var success = a.table.nodes.Any(n => n.nodeId == b.table.localNode.nodeId); - - if (log != null) + try { - var msg = success ? "PASS" : "FAIL"; - log.Log($"{msg} {a.table.localNode.nodeId} is looking for {b.table.localNode.nodeId} in table-nodes [{tableNodes}]"); + var response = a.Node.GetDebugPeer(peerId); + if (string.IsNullOrEmpty(response.peerId) || !response.addresses.Any()) + { + failureMessags.Add($"{a.Response.id} did not know peer {peerId}"); + } + else if (log != null) + { + log.Log($"{a.Response.id} knows {peerId}."); + } + } + catch (Exception e) + { + failureMessags.Add($"{a.Response.id} was unable to get 'debug/peer/{peerId}'. {e}"); } - //Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); - //Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); - if (!success) + ////var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); + ////var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); + //var tableNodes = string.Join(",", a.table.nodes.Select(n => n.nodeId)); + + //var success = a.table.nodes.Any(n => n.nodeId == b.table.localNode.nodeId); + + //if (log != null) + //{ + // var msg = success ? "PASS" : "FAIL"; + // log.Log($"{msg} {a.table.localNode.nodeId} is looking for {b.table.localNode.nodeId} in table-nodes [{tableNodes}]"); + //} + + ////Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); + ////Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); + //if (!success) + //{ + // failureMessags.Add($"{a.table.localNode.nodeId} was looking for '{b.table.localNode.nodeId}' in table-nodes [{tableNodes}] but it was not found."); + //} + } + + public class Entry + { + public Entry(IOnlineCodexNode node) { - failureMessags.Add($"{a.table.localNode.nodeId} was looking for '{b.table.localNode.nodeId}' in table-nodes [{tableNodes}] but it was not found."); + Node = node; + Response = node.GetDebugInfo(); } + + public IOnlineCodexNode Node { get ; } + public CodexDebugResponse Response { get; } } } } From f7e78494608d364d42859e5da5c811c6a3fa4f99 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 11 May 2023 13:59:53 +0200 Subject: [PATCH 06/16] Improves retrying in peer test helper --- Tests/PeerDiscoveryTests/PeerTestHelpers.cs | 47 +++++++-------------- 1 file changed, 15 insertions(+), 32 deletions(-) diff --git a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs index f607888..8b09fa6 100644 --- a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs +++ b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs @@ -15,21 +15,18 @@ namespace Tests.PeerDiscoveryTests public static void AssertFullyConnected(BaseLog? log = null, params IOnlineCodexNode[] nodes) { - Time.Retry(() => + var entries = nodes.Select(n => new Entry(n)).ToArray(); + + var failureMessags = new List(); + for (var x = 0; x < entries.Length; x++) { - var entries = nodes.Select(n => new Entry(n)).ToArray(); - - var failureMessags = new List(); - for (var x = 0; x < entries.Length; x++) + for (var y = x + 1; y < entries.Length; y++) { - for (var y = x + 1; y < entries.Length; y++) - { - AssertKnowEachother(failureMessags, entries[x], entries[y], log); - } + AssertKnowEachother(failureMessags, entries[x], entries[y], log); } + } - CollectionAssert.IsEmpty(failureMessags); - }); + CollectionAssert.IsEmpty(failureMessags); } private static void AssertKnowEachother(List failureMessags, Entry a, Entry b, BaseLog? log) @@ -44,7 +41,7 @@ namespace Tests.PeerDiscoveryTests try { - var response = a.Node.GetDebugPeer(peerId); + var response = GetDebugPeer(a.Node, peerId); if (string.IsNullOrEmpty(response.peerId) || !response.addresses.Any()) { failureMessags.Add($"{a.Response.id} did not know peer {peerId}"); @@ -54,29 +51,15 @@ namespace Tests.PeerDiscoveryTests log.Log($"{a.Response.id} knows {peerId}."); } } - catch (Exception e) + catch { - failureMessags.Add($"{a.Response.id} was unable to get 'debug/peer/{peerId}'. {e}"); + failureMessags.Add($"{a.Response.id} was unable to get 'debug/peer/{peerId}'."); } + } - ////var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); - ////var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); - //var tableNodes = string.Join(",", a.table.nodes.Select(n => n.nodeId)); - - //var success = a.table.nodes.Any(n => n.nodeId == b.table.localNode.nodeId); - - //if (log != null) - //{ - // var msg = success ? "PASS" : "FAIL"; - // log.Log($"{msg} {a.table.localNode.nodeId} is looking for {b.table.localNode.nodeId} in table-nodes [{tableNodes}]"); - //} - - ////Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); - ////Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); - //if (!success) - //{ - // failureMessags.Add($"{a.table.localNode.nodeId} was looking for '{b.table.localNode.nodeId}' in table-nodes [{tableNodes}] but it was not found."); - //} + private static CodexDebugPeerResponse GetDebugPeer(IOnlineCodexNode node, string peerId) + { + return Time.Retry(() => node.GetDebugPeer(peerId)); } public class Entry From 90a5aafa1cbb6a990188b6d30eeccb76457690f1 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 12 May 2023 09:11:05 +0200 Subject: [PATCH 07/16] Better peer discovery result logging. Staged discovery test. --- DistTestCore/Codex/CodexAccess.cs | 2 +- DistTestCore/Http.cs | 2 - .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 21 +++++---- Tests/PeerDiscoveryTests/PeerTestHelpers.cs | 44 ++++++++++++++----- 4 files changed, 47 insertions(+), 22 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 00affc2..5a8d472 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -24,7 +24,7 @@ namespace DistTestCore.Codex public CodexDebugPeerResponse GetDebugPeer(string peerId) { - return Http().HttpGetJson($"debug/peer/{peerId}"); + return Http(TimeSpan.FromSeconds(2)).HttpGetJson($"debug/peer/{peerId}"); } public string UploadFile(FileStream fileStream) diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index a4feeb8..a83df8a 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -1,6 +1,5 @@ using Logging; using Newtonsoft.Json; -using NUnit.Framework; using System.Net.Http.Headers; using System.Net.Http.Json; using Utils; @@ -120,7 +119,6 @@ namespace DistTestCore catch (Exception exception) { var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}"; - Assert.Fail(msg); throw new InvalidOperationException(msg, exception); } } diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 9d5c735..a2f3ed4 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -6,14 +6,6 @@ namespace Tests.PeerDiscoveryTests [TestFixture] public class PeerDiscoveryTests : AutoBootstrapDistTest { - [Test] - public void TwoNodes() - { - SetupCodexNode(); - - AssertAllNodesConnected(); - } - [TestCase(2)] [TestCase(3)] [TestCase(10)] @@ -37,6 +29,19 @@ namespace Tests.PeerDiscoveryTests AssertAllNodesConnected(); } + [TestCase(3, 3)] + [TestCase(3, 5)] + [TestCase(3, 10)] + public void StagedVariableNodes(int numberOfNodes, int numberOfStages) + { + for (var i = 0; i < numberOfStages; i++) + { + SetupCodexNodes(numberOfNodes); + + AssertAllNodesConnected(); + } + } + private void AssertAllNodesConnected() { PeerTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes(), GetTestLog()); diff --git a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs index 8b09fa6..99a6cbb 100644 --- a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs +++ b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs @@ -31,35 +31,57 @@ namespace Tests.PeerDiscoveryTests private static void AssertKnowEachother(List failureMessags, Entry a, Entry b, BaseLog? log) { - AssertKnows(failureMessags, a, b, log); - AssertKnows(failureMessags, b, a, log); + var aKnowsB = Knows(a, b); + var bKnowsA = Knows(b, a); + + var message = GetMessage(a, b, aKnowsB, bKnowsA); + + if (log != null) log.Log(message); + if (!aKnowsB || !bKnowsA) failureMessags.Add(message); } - private static void AssertKnows(List failureMessags, Entry a, Entry b, BaseLog? log) + private static string GetMessage(Entry a, Entry b, bool aKnowsB, bool bKnowsA) + { + var aName = a.Response.id; + var bName = b.Response.id; + + if (aKnowsB && bKnowsA) + { + return $"{aName} and {bName} know each other."; + } + if (aKnowsB) + { + return $"{aName} knows {bName}, but {bName} does not know {aName}"; + } + if (bKnowsA) + { + return $"{bName} knows {aName}, but {aName} does not know {bName}"; + } + return $"{aName} and {bName} don't know each other."; + } + + private static bool Knows(Entry a, Entry b) { var peerId = b.Response.id; try { var response = GetDebugPeer(a.Node, peerId); - if (string.IsNullOrEmpty(response.peerId) || !response.addresses.Any()) + if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any()) { - failureMessags.Add($"{a.Response.id} did not know peer {peerId}"); - } - else if (log != null) - { - log.Log($"{a.Response.id} knows {peerId}."); + return true; } } catch { - failureMessags.Add($"{a.Response.id} was unable to get 'debug/peer/{peerId}'."); } + + return false; } private static CodexDebugPeerResponse GetDebugPeer(IOnlineCodexNode node, string peerId) { - return Time.Retry(() => node.GetDebugPeer(peerId)); + return Time.Retry(() => node.GetDebugPeer(peerId), TimeSpan.FromMinutes(2), TimeSpan.FromSeconds(0.1)); } public class Entry From 830d74a5ae02fab6f8060eeee1fcf845dd392237 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 12 May 2023 10:48:12 +0200 Subject: [PATCH 08/16] Faster peer-pair checking --- Tests/PeerDiscoveryTests/PeerTestHelpers.cs | 174 +++++++++++++------- 1 file changed, 113 insertions(+), 61 deletions(-) diff --git a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs index 99a6cbb..c404fe3 100644 --- a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs +++ b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs @@ -1,8 +1,8 @@ using DistTestCore.Codex; using DistTestCore; using NUnit.Framework; -using Utils; using Logging; +using Utils; namespace Tests.PeerDiscoveryTests { @@ -15,73 +15,65 @@ namespace Tests.PeerDiscoveryTests public static void AssertFullyConnected(BaseLog? log = null, params IOnlineCodexNode[] nodes) { - var entries = nodes.Select(n => new Entry(n)).ToArray(); + var entries = CreateEntries(nodes); + var pairs = CreatePairs(entries); - var failureMessags = new List(); + RetryWhilePairs(pairs, () => + { + CheckAndRemoveSuccessful(pairs, log); + }); + + if (pairs.Any()) + { + Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage()))); + } + } + + private static void RetryWhilePairs(List pairs, Action action) + { + var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2); + while (pairs.Any() && (timeout > DateTime.UtcNow)) + { + action(); + + if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(5)); + } + } + + private static void CheckAndRemoveSuccessful(List pairs, BaseLog? log) + { + var checkTasks = pairs.Select(p => Task.Run(p.Check)).ToArray(); + Task.WaitAll(checkTasks); + + foreach (var pair in pairs.ToArray()) + { + if (pair.Success) + { + pairs.Remove(pair); + if (log != null) log.Log(pair.GetMessage()); + } + } + } + + private static Entry[] CreateEntries(IOnlineCodexNode[] nodes) + { + return nodes.Select(n => new Entry(n)).ToArray(); + } + + private static List CreatePairs(Entry[] entries) + { + return CreatePairsIterator(entries).ToList(); + } + + private static IEnumerable CreatePairsIterator(Entry[] entries) + { for (var x = 0; x < entries.Length; x++) { for (var y = x + 1; y < entries.Length; y++) { - AssertKnowEachother(failureMessags, entries[x], entries[y], log); + yield return new Pair(entries[x], entries[y]); } } - - CollectionAssert.IsEmpty(failureMessags); - } - - private static void AssertKnowEachother(List failureMessags, Entry a, Entry b, BaseLog? log) - { - var aKnowsB = Knows(a, b); - var bKnowsA = Knows(b, a); - - var message = GetMessage(a, b, aKnowsB, bKnowsA); - - if (log != null) log.Log(message); - if (!aKnowsB || !bKnowsA) failureMessags.Add(message); - } - - private static string GetMessage(Entry a, Entry b, bool aKnowsB, bool bKnowsA) - { - var aName = a.Response.id; - var bName = b.Response.id; - - if (aKnowsB && bKnowsA) - { - return $"{aName} and {bName} know each other."; - } - if (aKnowsB) - { - return $"{aName} knows {bName}, but {bName} does not know {aName}"; - } - if (bKnowsA) - { - return $"{bName} knows {aName}, but {aName} does not know {bName}"; - } - return $"{aName} and {bName} don't know each other."; - } - - private static bool Knows(Entry a, Entry b) - { - var peerId = b.Response.id; - - try - { - var response = GetDebugPeer(a.Node, peerId); - if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any()) - { - return true; - } - } - catch - { - } - - return false; - } - - private static CodexDebugPeerResponse GetDebugPeer(IOnlineCodexNode node, string peerId) - { - return Time.Retry(() => node.GetDebugPeer(peerId), TimeSpan.FromMinutes(2), TimeSpan.FromSeconds(0.1)); } public class Entry @@ -95,5 +87,65 @@ namespace Tests.PeerDiscoveryTests public IOnlineCodexNode Node { get ; } public CodexDebugResponse Response { get; } } + + public class Pair + { + public Pair(Entry a, Entry b) + { + A = a; + B = b; + } + + public Entry A { get; } + public Entry B { get; } + public bool AKnowsB { get; private set; } + public bool BKnowsA { get; private set; } + public bool Success { get { return AKnowsB && BKnowsA; } } + + public void Check() + { + AKnowsB = Knows(A, B); + BKnowsA = Knows(B, A); + } + + public string GetMessage() + { + var aName = A.Response.id; + var bName = B.Response.id; + + if (AKnowsB && BKnowsA) + { + return $"{aName} and {bName} know each other."; + } + if (AKnowsB) + { + return $"{aName} knows {bName}, but {bName} does not know {aName}"; + } + if (BKnowsA) + { + return $"{bName} knows {aName}, but {aName} does not know {bName}"; + } + return $"{aName} and {bName} don't know each other."; + } + + private static bool Knows(Entry a, Entry b) + { + var peerId = b.Response.id; + + try + { + var response = a.Node.GetDebugPeer(peerId); + if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any()) + { + return true; + } + } + catch + { + } + + return false; + } + } } } From 025b03e678169d6155a1ac1f90bd6f3a139f95e5 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 18 May 2023 10:42:04 +0200 Subject: [PATCH 09/16] Retain for a while in staged variable nodes peer discovery tests --- DistTestCore/AutoBootstrapDistTest.cs | 5 +- DistTestCore/Codex/CodexAccess.cs | 9 ++- DistTestCore/OnlineCodexNode.cs | 9 ++- .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 14 +++- Tests/PeerDiscoveryTests/PeerTestHelpers.cs | 73 +++++++++++++++---- 5 files changed, 92 insertions(+), 18 deletions(-) diff --git a/DistTestCore/AutoBootstrapDistTest.cs b/DistTestCore/AutoBootstrapDistTest.cs index 2ac0f8d..28e9763 100644 --- a/DistTestCore/AutoBootstrapDistTest.cs +++ b/DistTestCore/AutoBootstrapDistTest.cs @@ -21,7 +21,10 @@ namespace DistTestCore [SetUp] public void SetUpBootstrapNode() { - BootstrapNode = BringOnline(new CodexSetup(1))[0]; + BootstrapNode = BringOnline(new CodexSetup(1) + { + LogLevel = Codex.CodexLogLevel.Trace + })[0]; } protected IOnlineCodexNode BootstrapNode { get; private set; } = null!; diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 5a8d472..292cf46 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -24,7 +24,12 @@ namespace DistTestCore.Codex public CodexDebugPeerResponse GetDebugPeer(string peerId) { - return Http(TimeSpan.FromSeconds(2)).HttpGetJson($"debug/peer/{peerId}"); + return GetDebugPeer(peerId, TimeSpan.FromSeconds(2)); + } + + public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) + { + return Http(timeout).HttpGetJson($"debug/peer/{peerId}"); } public string UploadFile(FileStream fileStream) @@ -62,6 +67,7 @@ namespace DistTestCore.Codex var nodePeerId = debugInfo.id; var nodeName = Container.Name; log.AddStringReplace(nodePeerId, $"___{nodeName}___"); + log.AddStringReplace(debugInfo.table.localNode.nodeId, $"__{nodeName}__"); } catch (Exception e) { @@ -99,6 +105,7 @@ namespace DistTestCore.Codex public class CodexDebugTableNodeResponse { public string nodeId { get; set; } = string.Empty; + public string peerId { get; set; } = string.Empty; public string record { get; set; } = string.Empty; public bool seen { get; set; } } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index ccec842..322fb04 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -11,6 +11,7 @@ namespace DistTestCore string GetName(); CodexDebugResponse GetDebugInfo(); CodexDebugPeerResponse GetDebugPeer(string peerId); + CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout); ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); void ConnectToPeer(IOnlineCodexNode node); @@ -48,7 +49,8 @@ namespace DistTestCore public CodexDebugResponse GetDebugInfo() { var debugInfo = CodexAccess.GetDebugInfo(); - Log($"Got DebugInfo with id: '{debugInfo.id}'."); + var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId)); + Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}"); return debugInfo; } @@ -57,6 +59,11 @@ namespace DistTestCore return CodexAccess.GetDebugPeer(peerId); } + public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) + { + return CodexAccess.GetDebugPeer(peerId, timeout); + } + public ContentId UploadFile(TestFile file) { Log($"Uploading file of size {file.GetFileSize()}..."); diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index a2f3ed4..ad9b5c9 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -1,5 +1,7 @@ using DistTestCore; +using DistTestCore.Codex; using NUnit.Framework; +using Utils; namespace Tests.PeerDiscoveryTests { @@ -32,14 +34,24 @@ namespace Tests.PeerDiscoveryTests [TestCase(3, 3)] [TestCase(3, 5)] [TestCase(3, 10)] + [TestCase(5, 10)] + [TestCase(3, 20)] + [TestCase(5, 20)] public void StagedVariableNodes(int numberOfNodes, int numberOfStages) { for (var i = 0; i < numberOfStages; i++) { - SetupCodexNodes(numberOfNodes); + SetupCodexNodes(numberOfNodes, s => s.WithLogLevel(CodexLogLevel.Trace)); AssertAllNodesConnected(); } + + // Retain for a while + for (int i = 0; i < 10; i++) + { + Time.Sleep(TimeSpan.FromSeconds(30)); + AssertAllNodesConnected(); + } } private void AssertAllNodesConnected() diff --git a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs index c404fe3..d9b70a6 100644 --- a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs +++ b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs @@ -8,6 +8,8 @@ namespace Tests.PeerDiscoveryTests { public static class PeerTestHelpers { + private static readonly Random random = new Random(); + public static void AssertFullyConnected(IEnumerable nodes, BaseLog? log = null) { AssertFullyConnected(log, nodes.ToArray()); @@ -31,7 +33,7 @@ namespace Tests.PeerDiscoveryTests private static void RetryWhilePairs(List pairs, Action action) { - var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2); + var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5); while (pairs.Any() && (timeout > DateTime.UtcNow)) { action(); @@ -52,6 +54,10 @@ namespace Tests.PeerDiscoveryTests pairs.Remove(pair); if (log != null) log.Log(pair.GetMessage()); } + else + { + pair.IncreaseTimeout(); + } } } @@ -90,6 +96,10 @@ namespace Tests.PeerDiscoveryTests public class Pair { + private TimeSpan timeout = TimeSpan.FromSeconds(60); + private TimeSpan aToBTime = TimeSpan.FromSeconds(0); + private TimeSpan bToATime = TimeSpan.FromSeconds(0); + public Pair(Entry a, Entry b) { A = a; @@ -104,11 +114,22 @@ namespace Tests.PeerDiscoveryTests public void Check() { - AKnowsB = Knows(A, B); - BKnowsA = Knows(B, A); + ApplyRandomDelay(); + aToBTime = Measure(() => AKnowsB = Knows(A, B)); + bToATime = Measure(() => BKnowsA = Knows(B, A)); } public string GetMessage() + { + return GetResultMessage() + GetTimePostfix(); + } + + public void IncreaseTimeout() + { + //timeout *= 2; + } + + private string GetResultMessage() { var aName = A.Response.id; var bName = B.Response.id; @@ -128,23 +149,47 @@ namespace Tests.PeerDiscoveryTests return $"{aName} and {bName} don't know each other."; } - private static bool Knows(Entry a, Entry b) + private string GetTimePostfix() { - var peerId = b.Response.id; + var aName = A.Response.id; + var bName = B.Response.id; - try + return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)"; + } + + private static void ApplyRandomDelay() + { + // Calling all the nodes all at the same time is not exactly nice. + Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 100))); + } + + private static TimeSpan Measure(Action action) + { + var start = DateTime.UtcNow; + action(); + return DateTime.UtcNow - start; + } + + private bool Knows(Entry a, Entry b) + { + lock (a) { - var response = a.Node.GetDebugPeer(peerId); - if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any()) + var peerId = b.Response.id; + + try + { + var response = a.Node.GetDebugPeer(peerId, timeout); + if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any()) + { + return true; + } + } + catch { - return true; } - } - catch - { - } - return false; + return false; + } } } } From ab7a334987d5859a5106c7931a1307dfda219a2f Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 29 May 2023 08:35:46 +0200 Subject: [PATCH 10/16] Adds assert that nodes have correct addresses in their table of peers. --- DistTestCore/Codex/CodexAccess.cs | 5 +- DistTestCore/Codex/CodexContainerRecipe.cs | 3 +- .../LayeredDiscoveryTests.cs | 8 ++++ .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 3 +- Tests/PeerDiscoveryTests/PeerTestHelpers.cs | 46 ++++++++++++++----- 5 files changed, 48 insertions(+), 17 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 292cf46..b664bb9 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -66,8 +66,8 @@ namespace DistTestCore.Codex var nodePeerId = debugInfo.id; var nodeName = Container.Name; - log.AddStringReplace(nodePeerId, $"___{nodeName}___"); - log.AddStringReplace(debugInfo.table.localNode.nodeId, $"__{nodeName}__"); + log.AddStringReplace(nodePeerId, nodeName); + log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName); } catch (Exception e) { @@ -107,6 +107,7 @@ namespace DistTestCore.Codex public string nodeId { get; set; } = string.Empty; public string peerId { get; set; } = string.Empty; public string record { get; set; } = string.Empty; + public string address { get; set; } = string.Empty; public bool seen { get; set; } } diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index ff263a7..12023e8 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -13,6 +13,7 @@ namespace DistTestCore.Codex public const string DockerImage = "thatbenbierens/codexlocal:latest"; #endif public const string MetricsPortTag = "metrics_port"; + public const string DiscoveryPortTag = "discovery-port"; protected override string Image => DockerImage; @@ -22,7 +23,7 @@ namespace DistTestCore.Codex AddExposedPortAndVar("API_PORT"); AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}"); - AddInternalPortAndVar("DISC_PORT"); + AddInternalPortAndVar("DISC_PORT", DiscoveryPortTag); var listenPort = AddInternalPort(); AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); diff --git a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index 7028b23..5eaf1b4 100644 --- a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using NUnit.Framework; +using Utils; namespace Tests.PeerDiscoveryTests { @@ -33,6 +34,7 @@ namespace Tests.PeerDiscoveryTests [TestCase(5)] [TestCase(10)] [TestCase(20)] + [TestCase(50)] public void NodeChainTest(int chainLength) { var node = SetupCodexNode(); @@ -42,6 +44,12 @@ namespace Tests.PeerDiscoveryTests } AssertAllNodesConnected(); + + for (int i = 0; i < 5; i++) + { + Time.Sleep(TimeSpan.FromSeconds(30)); + AssertAllNodesConnected(); + } } private void AssertAllNodesConnected() diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index ad9b5c9..1df3e84 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -46,8 +46,7 @@ namespace Tests.PeerDiscoveryTests AssertAllNodesConnected(); } - // Retain for a while - for (int i = 0; i < 10; i++) + for (int i = 0; i < 5; i++) { Time.Sleep(TimeSpan.FromSeconds(30)); AssertAllNodesConnected(); diff --git a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs index d9b70a6..f3f4f0f 100644 --- a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs +++ b/Tests/PeerDiscoveryTests/PeerTestHelpers.cs @@ -52,18 +52,22 @@ namespace Tests.PeerDiscoveryTests if (pair.Success) { pairs.Remove(pair); - if (log != null) log.Log(pair.GetMessage()); - } - else - { - pair.IncreaseTimeout(); } } } private static Entry[] CreateEntries(IOnlineCodexNode[] nodes) { - return nodes.Select(n => new Entry(n)).ToArray(); + var entries = nodes.Select(n => new Entry(n)).ToArray(); + var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray(); + + if (incorrectDiscoveryEndpoints.Any()) + { + Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " + + string.Join(Environment.NewLine, incorrectDiscoveryEndpoints)); + } + + return entries; } private static List CreatePairs(Entry[] entries) @@ -92,11 +96,34 @@ namespace Tests.PeerDiscoveryTests public IOnlineCodexNode Node { get ; } public CodexDebugResponse Response { get; } + + public IEnumerable GetInCorrectDiscoveryEndpoints(Entry[] allEntries) + { + foreach (var peer in Response.table.nodes) + { + var expected = GetExpectedDiscoveryEndpoint(allEntries, peer); + if (expected != peer.address) + { + yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'"; + } + } + } + + private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node) + { + var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId); + if (peer == null) return $"peerId: {node.peerId} is not known."; + + var n = (OnlineCodexNode)peer.Node; + var ip = n.CodexAccess.Container.Pod.Ip; + var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag); + return $"{ip}:{discPort.Number}"; + } } public class Pair { - private TimeSpan timeout = TimeSpan.FromSeconds(60); + private readonly TimeSpan timeout = TimeSpan.FromSeconds(60); private TimeSpan aToBTime = TimeSpan.FromSeconds(0); private TimeSpan bToATime = TimeSpan.FromSeconds(0); @@ -124,11 +151,6 @@ namespace Tests.PeerDiscoveryTests return GetResultMessage() + GetTimePostfix(); } - public void IncreaseTimeout() - { - //timeout *= 2; - } - private string GetResultMessage() { var aName = A.Response.id; From 1016f568b81bb8d359fc37545ec550d3e6efa152 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 29 May 2023 09:13:38 +0200 Subject: [PATCH 11/16] Cleanup of peer connection helpers. Adds peer download helpers. --- DistTestCore/DistTest.cs | 24 +++++++- DistTestCore/FileManager.cs | 25 +++++++++ .../Helpers/PeerConnectionTestHelpers.cs | 55 +++++++++++-------- .../Helpers/PeerDownloadTestHelpers.cs | 49 +++++++++++++++++ .../LayeredDiscoveryTests.cs | 3 +- .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 3 +- 6 files changed, 131 insertions(+), 28 deletions(-) rename Tests/PeerDiscoveryTests/PeerTestHelpers.cs => DistTestCore/Helpers/PeerConnectionTestHelpers.cs (84%) create mode 100644 DistTestCore/Helpers/PeerDownloadTestHelpers.cs diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index f0280b1..4ca7e3b 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using DistTestCore.Helpers; using DistTestCore.Logs; using DistTestCore.Marketplace; using DistTestCore.Metrics; @@ -25,8 +26,14 @@ namespace DistTestCore testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray(); fixtureLog = new FixtureLog(configuration.GetLogConfig()); + + PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this); + PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this); } + public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; } + public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; } + [OneTimeSetUp] public void GlobalSetup() { @@ -85,6 +92,17 @@ namespace DistTestCore return Get().FileManager.GenerateTestFile(size); } + /// + /// Any test files generated in 'action' will be deleted after it returns. + /// This helps prevent large tests from filling up discs. + /// + public void ScopedTestFiles(Action action) + { + Get().FileManager.PushFileSet(); + action(); + Get().FileManager.PopFileSet(); + } + public IOnlineCodexNode SetupCodexBootstrapNode() { return SetupCodexBootstrapNode(s => { }); @@ -133,18 +151,18 @@ namespace DistTestCore return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes); } - protected BaseLog GetTestLog() + public BaseLog GetTestLog() { return Get().Log; } - protected void Log(string msg) + public void Log(string msg) { TestContext.Progress.WriteLine(msg); GetTestLog().Log(msg); } - protected void Debug(string msg) + public void Debug(string msg) { TestContext.Progress.WriteLine(msg); GetTestLog().Debug(msg); diff --git a/DistTestCore/FileManager.cs b/DistTestCore/FileManager.cs index ae58cd6..97cf27c 100644 --- a/DistTestCore/FileManager.cs +++ b/DistTestCore/FileManager.cs @@ -9,6 +9,8 @@ namespace DistTestCore TestFile CreateEmptyTestFile(); TestFile GenerateTestFile(ByteSize size); void DeleteAllTestFiles(); + void PushFileSet(); + void PopFileSet(); } public class FileManager : IFileManager @@ -18,6 +20,7 @@ namespace DistTestCore private readonly Random random = new Random(); private readonly TestLog log; private readonly string folder; + private readonly List> fileSetStack = new List>(); public FileManager(TestLog log, Configuration configuration) { @@ -31,6 +34,7 @@ namespace DistTestCore { var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin")); File.Create(result.Filename).Close(); + if (fileSetStack.Any()) fileSetStack.Last().Add(result); return result; } @@ -47,6 +51,27 @@ namespace DistTestCore DeleteDirectory(); } + public void PushFileSet() + { + fileSetStack.Add(new List()); + } + + public void PopFileSet() + { + if (!fileSetStack.Any()) return; + var pop = fileSetStack.Last(); + fileSetStack.Remove(pop); + + foreach (var file in pop) + { + try + { + File.Delete(file.Filename); + } + catch { } + } + } + private void GenerateFileBytes(TestFile result, ByteSize size) { long bytesLeft = size.SizeInBytes; diff --git a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs similarity index 84% rename from Tests/PeerDiscoveryTests/PeerTestHelpers.cs rename to DistTestCore/Helpers/PeerConnectionTestHelpers.cs index f3f4f0f..1f3422a 100644 --- a/Tests/PeerDiscoveryTests/PeerTestHelpers.cs +++ b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs @@ -1,30 +1,34 @@ using DistTestCore.Codex; -using DistTestCore; using NUnit.Framework; -using Logging; using Utils; -namespace Tests.PeerDiscoveryTests +namespace DistTestCore.Helpers { - public static class PeerTestHelpers + public class PeerConnectionTestHelpers { - private static readonly Random random = new Random(); + private readonly Random random = new Random(); + private readonly DistTest test; - public static void AssertFullyConnected(IEnumerable nodes, BaseLog? log = null) + public PeerConnectionTestHelpers(DistTest test) { - AssertFullyConnected(log, nodes.ToArray()); + this.test = test; } - public static void AssertFullyConnected(BaseLog? log = null, params IOnlineCodexNode[] nodes) + public void AssertFullyConnected(IEnumerable nodes) + { + AssertFullyConnected(nodes.ToArray()); + } + + public void AssertFullyConnected(params IOnlineCodexNode[] nodes) { var entries = CreateEntries(nodes); var pairs = CreatePairs(entries); RetryWhilePairs(pairs, () => { - CheckAndRemoveSuccessful(pairs, log); + CheckAndRemoveSuccessful(pairs); }); - + if (pairs.Any()) { Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage()))); @@ -34,7 +38,7 @@ namespace Tests.PeerDiscoveryTests private static void RetryWhilePairs(List pairs, Action action) { var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5); - while (pairs.Any() && (timeout > DateTime.UtcNow)) + while (pairs.Any() && timeout > DateTime.UtcNow) { action(); @@ -42,15 +46,21 @@ namespace Tests.PeerDiscoveryTests } } - private static void CheckAndRemoveSuccessful(List pairs, BaseLog? log) + private void CheckAndRemoveSuccessful(List pairs) { - var checkTasks = pairs.Select(p => Task.Run(p.Check)).ToArray(); + var checkTasks = pairs.Select(p => Task.Run(() => + { + ApplyRandomDelay(); + p.Check(); + })).ToArray(); + Task.WaitAll(checkTasks); foreach (var pair in pairs.ToArray()) { if (pair.Success) { + test.Debug(pair.GetMessage()); pairs.Remove(pair); } } @@ -60,7 +70,7 @@ namespace Tests.PeerDiscoveryTests { var entries = nodes.Select(n => new Entry(n)).ToArray(); var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray(); - + if (incorrectDiscoveryEndpoints.Any()) { Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " + @@ -86,6 +96,12 @@ namespace Tests.PeerDiscoveryTests } } + private void ApplyRandomDelay() + { + // Calling all the nodes all at the same time is not exactly nice. + Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 100))); + } + public class Entry { public Entry(IOnlineCodexNode node) @@ -94,7 +110,7 @@ namespace Tests.PeerDiscoveryTests Response = node.GetDebugInfo(); } - public IOnlineCodexNode Node { get ; } + public IOnlineCodexNode Node { get; } public CodexDebugResponse Response { get; } public IEnumerable GetInCorrectDiscoveryEndpoints(Entry[] allEntries) @@ -137,11 +153,10 @@ namespace Tests.PeerDiscoveryTests public Entry B { get; } public bool AKnowsB { get; private set; } public bool BKnowsA { get; private set; } - public bool Success { get { return AKnowsB && BKnowsA; } } + public bool Success { get { return AKnowsB && BKnowsA; } } public void Check() { - ApplyRandomDelay(); aToBTime = Measure(() => AKnowsB = Knows(A, B)); bToATime = Measure(() => BKnowsA = Knows(B, A)); } @@ -179,12 +194,6 @@ namespace Tests.PeerDiscoveryTests return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)"; } - private static void ApplyRandomDelay() - { - // Calling all the nodes all at the same time is not exactly nice. - Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 100))); - } - private static TimeSpan Measure(Action action) { var start = DateTime.UtcNow; diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs new file mode 100644 index 0000000..77e8901 --- /dev/null +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -0,0 +1,49 @@ +namespace DistTestCore.Helpers +{ + public class PeerDownloadTestHelpers + { + private readonly DistTest test; + + public PeerDownloadTestHelpers(DistTest test) + { + this.test = test; + } + + public void AssertFullDownloadInterconnectivity(IEnumerable nodes) + { + AssertFullDownloadInterconnectivity(nodes, 1.MB()); + } + + public void AssertFullDownloadInterconnectivity(IEnumerable nodes, ByteSize testFileSize) + { + foreach (var node in nodes) + { + var uploader = node; + var downloaders = nodes.Where(n => n != uploader).ToArray(); + + test.ScopedTestFiles(() => + { + PerformTest(uploader, downloaders); + }); + } + } + + private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders) + { + // 1 test file per downloader. + var files = downloaders.Select(d => test.GenerateTestFile(1.MB())).ToArray(); + + // Upload all the test files to the uploader. + var contentIds = files.Select(uploader.UploadFile).ToArray(); + + // Each downloader should retrieve its own test file. + for (var i = 0; i < downloaders.Length; i++) + { + var expectedFile = files[i]; + var downloadedFile = downloaders[i].DownloadContent(contentIds[i]); + + expectedFile.AssertIsEqual(downloadedFile); + } + } + } +} diff --git a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index 5eaf1b4..ad04144 100644 --- a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -1,4 +1,5 @@ using DistTestCore; +using DistTestCore.Helpers; using NUnit.Framework; using Utils; @@ -54,7 +55,7 @@ namespace Tests.PeerDiscoveryTests private void AssertAllNodesConnected() { - PeerTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes(), GetTestLog()); + PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); } } } diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 1df3e84..7797764 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -1,5 +1,6 @@ using DistTestCore; using DistTestCore.Codex; +using DistTestCore.Helpers; using NUnit.Framework; using Utils; @@ -55,7 +56,7 @@ namespace Tests.PeerDiscoveryTests private void AssertAllNodesConnected() { - PeerTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes(), GetTestLog()); + PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); } } } From 8c85cd22bb82d9c57369a8600290a2276acb5b93 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 29 May 2023 09:25:03 +0200 Subject: [PATCH 12/16] Adds clear logging for peer helper invocations. --- DistTestCore/Helpers/PeerConnectionTestHelpers.cs | 2 ++ DistTestCore/Helpers/PeerDownloadTestHelpers.cs | 3 +++ Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs | 1 + Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs | 1 + 4 files changed, 7 insertions(+) diff --git a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs index 1f3422a..9d6203f 100644 --- a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs +++ b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs @@ -21,6 +21,7 @@ namespace DistTestCore.Helpers public void AssertFullyConnected(params IOnlineCodexNode[] nodes) { + test.Debug($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); var entries = CreateEntries(nodes); var pairs = CreatePairs(entries); @@ -33,6 +34,7 @@ namespace DistTestCore.Helpers { Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage()))); } + test.Debug($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}"); } private static void RetryWhilePairs(List pairs, Action action) diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs index 77e8901..d3280e9 100644 --- a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -16,6 +16,7 @@ public void AssertFullDownloadInterconnectivity(IEnumerable nodes, ByteSize testFileSize) { + test.Debug($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); foreach (var node in nodes) { var uploader = node; @@ -26,6 +27,8 @@ PerformTest(uploader, downloaders); }); } + + test.Debug($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}"); } private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders) diff --git a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index ad04144..8e6dbf4 100644 --- a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -56,6 +56,7 @@ namespace Tests.PeerDiscoveryTests private void AssertAllNodesConnected() { PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); + PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes()); } } } diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 7797764..c5a85a4 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -57,6 +57,7 @@ namespace Tests.PeerDiscoveryTests private void AssertAllNodesConnected() { PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); + PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes()); } } } From 816cd1728bc27e63d179606705ce4bb79b39cc60 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 29 May 2023 09:27:25 +0200 Subject: [PATCH 13/16] wires up testfile size in download helper --- DistTestCore/Helpers/PeerDownloadTestHelpers.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs index d3280e9..c52d192 100644 --- a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -24,17 +24,17 @@ test.ScopedTestFiles(() => { - PerformTest(uploader, downloaders); + PerformTest(uploader, downloaders, testFileSize); }); } test.Debug($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}"); } - private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders) + private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize) { // 1 test file per downloader. - var files = downloaders.Select(d => test.GenerateTestFile(1.MB())).ToArray(); + var files = downloaders.Select(d => test.GenerateTestFile(testFileSize)).ToArray(); // Upload all the test files to the uploader. var contentIds = files.Select(uploader.UploadFile).ToArray(); From f2145a747f727664cb0c4a15855d240736eb42da Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 31 May 2023 13:15:41 +0200 Subject: [PATCH 14/16] Clearly distinguishes between peer-connections that can't be made and problems with the node-to-test link. --- DistTestCore/Codex/CodexAccess.cs | 17 +++++- .../Helpers/PeerConnectionTestHelpers.cs | 52 +++++++++++-------- .../Helpers/PeerDownloadTestHelpers.cs | 4 +- DistTestCore/Http.cs | 40 +++++++------- .../Marketplace/ContainerInfoExtractor.cs | 2 +- .../LayeredDiscoveryTests.cs | 2 +- .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 13 ++++- Utils/Time.cs | 24 ++++----- 8 files changed, 95 insertions(+), 59 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index b664bb9..17a468c 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -29,7 +29,20 @@ namespace DistTestCore.Codex public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) { - return Http(timeout).HttpGetJson($"debug/peer/{peerId}"); + var http = Http(timeout); + var str = http.HttpGetString($"debug/peer/{peerId}"); + + if (str.ToLowerInvariant() == "unable to find peer!") + { + return new CodexDebugPeerResponse + { + IsPeerFound = false + }; + } + + var result = http.TryJsonDeserialize(str); + result.IsPeerFound = true; + return result; } public string UploadFile(FileStream fileStream) @@ -139,6 +152,8 @@ namespace DistTestCore.Codex public class CodexDebugPeerResponse { + public bool IsPeerFound { get; set; } + public string peerId { get; set; } = string.Empty; public long seqNo { get; set; } public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty(); diff --git a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs index 9d6203f..92114bc 100644 --- a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs +++ b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs @@ -21,7 +21,7 @@ namespace DistTestCore.Helpers public void AssertFullyConnected(params IOnlineCodexNode[] nodes) { - test.Debug($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); + test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); var entries = CreateEntries(nodes); var pairs = CreatePairs(entries); @@ -32,14 +32,19 @@ namespace DistTestCore.Helpers if (pairs.Any()) { + test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}"); Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage()))); + test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage()))); + } + else + { + test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}"); } - test.Debug($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}"); } private static void RetryWhilePairs(List pairs, Action action) { - var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5); + var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(10); while (pairs.Any() && timeout > DateTime.UtcNow) { action(); @@ -62,7 +67,7 @@ namespace DistTestCore.Helpers { if (pair.Success) { - test.Debug(pair.GetMessage()); + test.Log(pair.GetMessage()); pairs.Remove(pair); } } @@ -101,7 +106,7 @@ namespace DistTestCore.Helpers private void ApplyRandomDelay() { // Calling all the nodes all at the same time is not exactly nice. - Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 100))); + Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000))); } public class Entry @@ -139,6 +144,13 @@ namespace DistTestCore.Helpers } } + public enum PeerConnectionState + { + Unknown, + Connection, + NoConnection, + } + public class Pair { private readonly TimeSpan timeout = TimeSpan.FromSeconds(60); @@ -153,9 +165,9 @@ namespace DistTestCore.Helpers public Entry A { get; } public Entry B { get; } - public bool AKnowsB { get; private set; } - public bool BKnowsA { get; private set; } - public bool Success { get { return AKnowsB && BKnowsA; } } + public PeerConnectionState AKnowsB { get; private set; } + public PeerConnectionState BKnowsA { get; private set; } + public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } } public void Check() { @@ -173,19 +185,12 @@ namespace DistTestCore.Helpers var aName = A.Response.id; var bName = B.Response.id; - if (AKnowsB && BKnowsA) + if (Success) { return $"{aName} and {bName} know each other."; } - if (AKnowsB) - { - return $"{aName} knows {bName}, but {bName} does not know {aName}"; - } - if (BKnowsA) - { - return $"{bName} knows {aName}, but {aName} does not know {bName}"; - } - return $"{aName} and {bName} don't know each other."; + + return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}"; } private string GetTimePostfix() @@ -203,7 +208,7 @@ namespace DistTestCore.Helpers return DateTime.UtcNow - start; } - private bool Knows(Entry a, Entry b) + private PeerConnectionState Knows(Entry a, Entry b) { lock (a) { @@ -212,16 +217,21 @@ namespace DistTestCore.Helpers try { var response = a.Node.GetDebugPeer(peerId, timeout); + if (!response.IsPeerFound) + { + return PeerConnectionState.NoConnection; + } if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any()) { - return true; + return PeerConnectionState.Connection; } } catch { } - return false; + // Didn't get a conclusive answer. Try again later. + return PeerConnectionState.Unknown; } } } diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs index c52d192..6929ceb 100644 --- a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -16,7 +16,7 @@ public void AssertFullDownloadInterconnectivity(IEnumerable nodes, ByteSize testFileSize) { - test.Debug($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); + test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); foreach (var node in nodes) { var uploader = node; @@ -28,7 +28,7 @@ }); } - test.Debug($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}"); + test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}"); } private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize) diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index a83df8a..955e4ef 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -38,7 +38,7 @@ namespace DistTestCore var str = Time.Wait(result.Content.ReadAsStringAsync()); Log(url, str); return str; ; - }); + }, $"HTTP-GET:{route}"); } public T HttpGetJson(string route) @@ -62,10 +62,10 @@ namespace DistTestCore using var content = JsonContent.Create(body); Log(url, JsonConvert.SerializeObject(body)); var result = Time.Wait(client.PostAsync(url, content)); - var str= Time.Wait(result.Content.ReadAsStringAsync()); + var str = Time.Wait(result.Content.ReadAsStringAsync()); Log(url, str); return str; - }); + }, $"HTTP-POST-JSON: {route}"); } public string HttpPostStream(string route, Stream stream) @@ -81,7 +81,7 @@ namespace DistTestCore var str =Time.Wait(response.Content.ReadAsStringAsync()); Log(url, str); return str; - }); + }, $"HTTP-POST-STREAM: {route}"); } public Stream HttpGetStream(string route) @@ -92,7 +92,20 @@ namespace DistTestCore var url = GetUrl() + route; Log(url, "~ STREAM ~"); return Time.Wait(client.GetStreamAsync(url)); - }); + }, $"HTTP-GET-STREAM: {route}"); + } + + public T TryJsonDeserialize(string json) + { + try + { + return JsonConvert.DeserializeObject(json)!; + } + catch (Exception exception) + { + var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}"; + throw new InvalidOperationException(msg, exception); + } } private string GetUrl() @@ -105,22 +118,9 @@ namespace DistTestCore log.Debug($"({url}) = '{message}'", 3); } - private T Retry(Func operation) + private T Retry(Func operation, string description) { - return Time.Retry(operation, timeSet.HttpCallRetryTimeout(), timeSet.HttpCallRetryDelay()); - } - - private static T TryJsonDeserialize(string json) - { - try - { - return JsonConvert.DeserializeObject(json)!; - } - catch (Exception exception) - { - var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}"; - throw new InvalidOperationException(msg, exception); - } + return Time.Retry(operation, timeSet.HttpCallRetryTimeout(), timeSet.HttpCallRetryDelay(), description); } private HttpClient GetClient() diff --git a/DistTestCore/Marketplace/ContainerInfoExtractor.cs b/DistTestCore/Marketplace/ContainerInfoExtractor.cs index 5f6db11..1aadc44 100644 --- a/DistTestCore/Marketplace/ContainerInfoExtractor.cs +++ b/DistTestCore/Marketplace/ContainerInfoExtractor.cs @@ -95,7 +95,7 @@ namespace DistTestCore.Marketplace private static string Retry(Func fetch) { - return Time.Retry(fetch); + return Time.Retry(fetch, nameof(ContainerInfoExtractor)); } } diff --git a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index 8e6dbf4..306d305 100644 --- a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -56,7 +56,7 @@ namespace Tests.PeerDiscoveryTests private void AssertAllNodesConnected() { PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); - PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes()); + //PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes()); } } } diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index c5a85a4..66c5ad6 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -9,6 +9,16 @@ namespace Tests.PeerDiscoveryTests [TestFixture] public class PeerDiscoveryTests : AutoBootstrapDistTest { + [Test] + public void CanReportUnknownPeerId() + { + var unknownId = "16Uiu2HAkv2CHWpff3dj5iuVNERAp8AGKGNgpGjPexJZHSqUstfsK"; + var node = SetupCodexNode(); + + var result = node.GetDebugPeer(unknownId); + Assert.That(result.IsPeerFound, Is.False); + } + [TestCase(2)] [TestCase(3)] [TestCase(10)] @@ -22,6 +32,7 @@ namespace Tests.PeerDiscoveryTests [TestCase(2)] [TestCase(3)] [TestCase(10)] + [TestCase(20)] public void VariableNodesInPods(int number) { for (var i = 0; i < number; i++) @@ -57,7 +68,7 @@ namespace Tests.PeerDiscoveryTests private void AssertAllNodesConnected() { PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); - PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes()); + //PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes()); } } } diff --git a/Utils/Time.cs b/Utils/Time.cs index 2002b9c..6ae2640 100644 --- a/Utils/Time.cs +++ b/Utils/Time.cs @@ -39,27 +39,27 @@ } } - public static void Retry(Action action) + public static void Retry(Action action, string description) { - Retry(action, TimeSpan.FromMinutes(1)); + Retry(action, TimeSpan.FromMinutes(1), description); } - public static T Retry(Func action) + public static T Retry(Func action, string description) { - return Retry(action, TimeSpan.FromMinutes(1)); + return Retry(action, TimeSpan.FromMinutes(1), description); } - public static void Retry(Action action, TimeSpan timeout) + public static void Retry(Action action, TimeSpan timeout, string description) { - Retry(action, timeout, TimeSpan.FromSeconds(1)); + Retry(action, timeout, TimeSpan.FromSeconds(1), description); } - public static T Retry(Func action, TimeSpan timeout) + public static T Retry(Func action, TimeSpan timeout, string description) { - return Retry(action, timeout, TimeSpan.FromSeconds(1)); + return Retry(action, timeout, TimeSpan.FromSeconds(1), description); } - public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime) + public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime, string description) { var start = DateTime.UtcNow; var exceptions = new List(); @@ -67,7 +67,7 @@ { if (DateTime.UtcNow - start > timeout) { - throw new TimeoutException("Retry timed out.", new AggregateException(exceptions)); + throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions)); } try @@ -84,7 +84,7 @@ } } - public static T Retry(Func action, TimeSpan timeout, TimeSpan retryTime) + public static T Retry(Func action, TimeSpan timeout, TimeSpan retryTime, string description) { var start = DateTime.UtcNow; var exceptions = new List(); @@ -92,7 +92,7 @@ { if (DateTime.UtcNow - start > timeout) { - throw new TimeoutException("Retry timed out.", new AggregateException(exceptions)); + throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions)); } try From 3d1d82f5825eefdb5c09805231996de6b57c001e Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 31 May 2023 13:34:12 +0200 Subject: [PATCH 15/16] pushes codex loglevel to configuration --- DistTestCore/AutoBootstrapDistTest.cs | 7 ++----- DistTestCore/Codex/CodexStartupConfig.cs | 7 ++++++- DistTestCore/CodexSetup.cs | 12 +++--------- DistTestCore/Configuration.cs | 8 +++++++- DistTestCore/DistTest.cs | 7 ++++++- LongTests/BasicTests/LargeFileTests.cs | 2 -- LongTests/BasicTests/TestInfraTests.cs | 2 -- Tests/BasicTests/ExampleTests.cs | 4 +--- Tests/DurabilityTests/DurabilityTests.cs | 6 +++--- Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs | 2 +- 10 files changed, 29 insertions(+), 28 deletions(-) diff --git a/DistTestCore/AutoBootstrapDistTest.cs b/DistTestCore/AutoBootstrapDistTest.cs index 28e9763..edf182f 100644 --- a/DistTestCore/AutoBootstrapDistTest.cs +++ b/DistTestCore/AutoBootstrapDistTest.cs @@ -12,7 +12,7 @@ namespace DistTestCore public override ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action setup) { - var codexSetup = new CodexSetup(numberOfNodes); + var codexSetup = CreateCodexSetup(numberOfNodes); setup(codexSetup); codexSetup.WithBootstrapNode(BootstrapNode); return BringOnline(codexSetup); @@ -21,10 +21,7 @@ namespace DistTestCore [SetUp] public void SetUpBootstrapNode() { - BootstrapNode = BringOnline(new CodexSetup(1) - { - LogLevel = Codex.CodexLogLevel.Trace - })[0]; + BootstrapNode = BringOnline(CreateCodexSetup(1))[0]; } protected IOnlineCodexNode BootstrapNode { get; private set; } = null!; diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index 9b3cb8a..b13512b 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -5,9 +5,14 @@ namespace DistTestCore.Codex { public class CodexStartupConfig { + public CodexStartupConfig(CodexLogLevel logLevel) + { + LogLevel = logLevel; + } + public string? NameOverride { get; set; } public Location Location { get; set; } - public CodexLogLevel? LogLevel { get; set; } + public CodexLogLevel LogLevel { get; } public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } public MarketplaceInitialConfig? MarketplaceConfig { get; set; } diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 5b5f3c0..83c5b9b 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -8,7 +8,6 @@ namespace DistTestCore { ICodexSetup WithName(string name); ICodexSetup At(Location location); - ICodexSetup WithLogLevel(CodexLogLevel level); ICodexSetup WithBootstrapNode(IOnlineCodexNode node); ICodexSetup WithStorageQuota(ByteSize storageQuota); ICodexSetup EnableMetrics(); @@ -20,7 +19,8 @@ namespace DistTestCore { public int NumberOfNodes { get; } - public CodexSetup(int numberOfNodes) + public CodexSetup(int numberOfNodes, CodexLogLevel logLevel) + : base(logLevel) { NumberOfNodes = numberOfNodes; } @@ -43,12 +43,6 @@ namespace DistTestCore return this; } - public ICodexSetup WithLogLevel(CodexLogLevel level) - { - LogLevel = level; - return this; - } - public ICodexSetup WithStorageQuota(ByteSize storageQuota) { StorageQuota = storageQuota; @@ -80,7 +74,7 @@ namespace DistTestCore private IEnumerable DescribeArgs() { - if (LogLevel != null) yield return $"LogLevel={LogLevel}"; + yield return $"LogLevel={LogLevel}"; if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}"; if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}"; } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index ee01559..ac7d470 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -1,4 +1,5 @@ -using KubernetesWorkflow; +using DistTestCore.Codex; +using KubernetesWorkflow; namespace DistTestCore { @@ -28,5 +29,10 @@ namespace DistTestCore { return "TestDataFiles"; } + + public CodexLogLevel GetCodexLogLevel() + { + return CodexLogLevel.Trace; + } } } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 4ca7e3b..bf4d59f 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -134,7 +134,7 @@ namespace DistTestCore public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action setup) { - var codexSetup = new CodexSetup(numberOfNodes); + var codexSetup = CreateCodexSetup(numberOfNodes); setup(codexSetup); @@ -168,6 +168,11 @@ namespace DistTestCore GetTestLog().Debug(msg); } + protected CodexSetup CreateCodexSetup(int numberOfNodes) + { + return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel()); + } + private TestLifecycle Get() { lock (lifecycleLock) diff --git a/LongTests/BasicTests/LargeFileTests.cs b/LongTests/BasicTests/LargeFileTests.cs index 63b33f5..3149ee3 100644 --- a/LongTests/BasicTests/LargeFileTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -1,5 +1,4 @@ using DistTestCore; -using DistTestCore.Codex; using NUnit.Framework; namespace TestsLong.BasicTests @@ -11,7 +10,6 @@ namespace TestsLong.BasicTests public void OneClientLargeFileTest() { var primary = SetupCodexNode(s => s - .WithLogLevel(CodexLogLevel.Warn) .WithStorageQuota(20.GB())); var testFile = GenerateTestFile(10.GB()); diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs index 0b5e640..9b3111f 100644 --- a/LongTests/BasicTests/TestInfraTests.cs +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -1,5 +1,4 @@ using DistTestCore; -using DistTestCore.Codex; using NUnit.Framework; namespace TestsLong.BasicTests @@ -32,7 +31,6 @@ namespace TestsLong.BasicTests public void DownloadConsistencyTest() { var primary = SetupCodexNode(s => s - .WithLogLevel(CodexLogLevel.Trace) .WithStorageQuota(2.MB())); var testFile = GenerateTestFile(1.MB()); diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 6951433..3a4884c 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -11,7 +11,7 @@ namespace Tests.BasicTests [Test] public void CodexLogExample() { - var primary = SetupCodexNode(s => s.WithLogLevel(CodexLogLevel.Trace)); + var primary = SetupCodexNode(); primary.UploadFile(GenerateTestFile(5.MB())); @@ -47,7 +47,6 @@ namespace Tests.BasicTests var buyerInitialBalance = 1000.TestTokens(); var seller = SetupCodexNode(s => s - .WithLogLevel(CodexLogLevel.Trace) .WithStorageQuota(11.GB()) .EnableMarketplace(sellerInitialBalance)); @@ -61,7 +60,6 @@ namespace Tests.BasicTests var testFile = GenerateTestFile(10.MB()); var buyer = SetupCodexNode(s => s - .WithLogLevel(CodexLogLevel.Trace) .WithBootstrapNode(seller) .EnableMarketplace(buyerInitialBalance)); diff --git a/Tests/DurabilityTests/DurabilityTests.cs b/Tests/DurabilityTests/DurabilityTests.cs index 008799b..1267400 100644 --- a/Tests/DurabilityTests/DurabilityTests.cs +++ b/Tests/DurabilityTests/DurabilityTests.cs @@ -31,10 +31,10 @@ namespace Tests.DurabilityTests [Test] public void DataRetentionTest() { - var bootstrapNode = SetupCodexBootstrapNode(s => s.WithLogLevel(CodexLogLevel.Trace)); + var bootstrapNode = SetupCodexBootstrapNode(); - var startGroup = SetupCodexNodes(2, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode)); - var finishGroup = SetupCodexNodes(10, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode)); + var startGroup = SetupCodexNodes(2, s => s.WithBootstrapNode(bootstrapNode)); + var finishGroup = SetupCodexNodes(10, s => s.WithBootstrapNode(bootstrapNode)); var file = GenerateTestFile(10.MB()); diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 66c5ad6..41b0a66 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -53,7 +53,7 @@ namespace Tests.PeerDiscoveryTests { for (var i = 0; i < numberOfStages; i++) { - SetupCodexNodes(numberOfNodes, s => s.WithLogLevel(CodexLogLevel.Trace)); + SetupCodexNodes(numberOfNodes); AssertAllNodesConnected(); } From 3ecdce7b079480558f97419c56dcb272bf59fce0 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 31 May 2023 13:50:52 +0200 Subject: [PATCH 16/16] Readies k8sCluster class for run against online cluster. --- DistTestCore/Codex/CodexAccess.cs | 2 +- DistTestCore/Codex/CodexContainerRecipe.cs | 6 +----- DistTestCore/Marketplace/GethBootstrapNodeInfo.cs | 2 +- DistTestCore/Marketplace/GethCompanionNodeInfo.cs | 2 +- DistTestCore/Metrics/MetricsQuery.cs | 2 +- KubernetesWorkflow/K8sCluster.cs | 15 +++++++++++---- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 17a468c..587f7f0 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -91,7 +91,7 @@ namespace DistTestCore.Codex private Http Http(TimeSpan? timeoutOverride = null) { - var ip = Container.Pod.Cluster.IP; + var ip = Container.Pod.Cluster.HostAddress; var port = Container.ServicePorts[0].Number; return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1", timeoutOverride); } diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index 12023e8..834f7af 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -24,6 +24,7 @@ namespace DistTestCore.Codex AddExposedPortAndVar("API_PORT"); AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}"); AddInternalPortAndVar("DISC_PORT", DiscoveryPortTag); + AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); var listenPort = AddInternalPort(); AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); @@ -32,11 +33,6 @@ namespace DistTestCore.Codex { AddEnvVar("BOOTSTRAP_SPR", config.BootstrapSpr); } - - if (config.LogLevel != null) - { - AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); - } if (config.StorageQuota != null) { AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!); diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index b59fb80..ab38cb1 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -23,7 +23,7 @@ namespace DistTestCore.Marketplace public NethereumInteraction StartInteraction(BaseLog log) { - var ip = RunningContainers.RunningPod.Cluster.IP; + var ip = RunningContainers.RunningPod.Cluster.HostAddress; var port = RunningContainers.Containers[0].ServicePorts[0].Number; var account = Account; diff --git a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs index 5731ab3..6c4b8e8 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs @@ -17,7 +17,7 @@ namespace DistTestCore.Marketplace public NethereumInteraction StartInteraction(BaseLog log, GethAccount account) { - var ip = RunningContainer.Pod.Cluster.IP; + var ip = RunningContainer.Pod.Cluster.HostAddress; var port = RunningContainer.ServicePorts[0].Number; var privateKey = account.PrivateKey; diff --git a/DistTestCore/Metrics/MetricsQuery.cs b/DistTestCore/Metrics/MetricsQuery.cs index 8c5f24f..f162690 100644 --- a/DistTestCore/Metrics/MetricsQuery.cs +++ b/DistTestCore/Metrics/MetricsQuery.cs @@ -16,7 +16,7 @@ namespace DistTestCore.Metrics http = new Http( log, timeSet, - runningContainers.RunningPod.Cluster.IP, + runningContainers.RunningPod.Cluster.HostAddress, runningContainers.Containers[0].ServicePorts[0].Number, "api/v1"); } diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs index 4d5a772..9ad7a6a 100644 --- a/KubernetesWorkflow/K8sCluster.cs +++ b/KubernetesWorkflow/K8sCluster.cs @@ -10,12 +10,12 @@ namespace KubernetesWorkflow } public Configuration Configuration { get; } - public string IP { get; private set; } = string.Empty; + public string HostAddress { get; private set; } = string.Empty; public KubernetesClientConfiguration GetK8sClientConfig() { var config = GetConfig(); - UpdateIp(config); + UpdateHostAddress(config); return config; } @@ -47,10 +47,17 @@ namespace KubernetesWorkflow } } - private void UpdateIp(KubernetesClientConfiguration config) + private void UpdateHostAddress(KubernetesClientConfiguration config) { var host = config.Host.Replace("https://", ""); - IP = host.Substring(0, host.IndexOf(':')); + if (host.Contains(":")) + { + HostAddress = host.Substring(0, host.IndexOf(':')); + } + else + { + HostAddress = config.Host; + } } } }