diff --git a/CodexPlugin/CoreInterfaceExtensions.cs b/CodexPlugin/CoreInterfaceExtensions.cs index dce2bcc..58b1920 100644 --- a/CodexPlugin/CoreInterfaceExtensions.cs +++ b/CodexPlugin/CoreInterfaceExtensions.cs @@ -15,6 +15,11 @@ namespace CodexPlugin return Plugin(ci).WrapCodexContainers(containers); } + public static IOnlineCodexNode SetupCodexNode(this CoreInterface ci) + { + return Plugin(ci).SetupCodexNode(s => { }); // do more unification here. Keep plugin simpler. + } + public static IOnlineCodexNode SetupCodexNode(this CoreInterface ci, Action setup) { return Plugin(ci).SetupCodexNode(setup); diff --git a/DistTestCore/AutoBootstrapDistTest.cs b/DistTestCore/AutoBootstrapDistTest.cs deleted file mode 100644 index 6fadf9b..0000000 --- a/DistTestCore/AutoBootstrapDistTest.cs +++ /dev/null @@ -1,30 +0,0 @@ -using NUnit.Framework; - -namespace DistTestCore -{ - public class AutoBootstrapDistTest : DistTest - { - //public override IOnlineCodexNode SetupCodexBootstrapNode(Action setup) - //{ - // throw new Exception("AutoBootstrapDistTest creates and attaches a single bootstrap node for you. " + - // "If you want to control the bootstrap node from your test, please use DistTest instead."); - //} - - //public override ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action setup) - //{ - // var codexSetup = CreateCodexSetup(numberOfNodes); - // setup(codexSetup); - // codexSetup.WithBootstrapNode(BootstrapNode); - // return BringOnline(codexSetup); - //} - - //[SetUp] - //public void SetUpBootstrapNode() - //{ - // var setup = CreateCodexSetup(1).WithName("BOOTSTRAP"); - // BootstrapNode = BringOnline(setup)[0]; - //} - - //protected IOnlineCodexNode BootstrapNode { get; private set; } = null!; - } -} diff --git a/DistTestCore/DownloadedLogExtensions.cs b/DistTestCore/DownloadedLogExtensions.cs new file mode 100644 index 0000000..59b020a --- /dev/null +++ b/DistTestCore/DownloadedLogExtensions.cs @@ -0,0 +1,13 @@ +using Core; +using NUnit.Framework; + +namespace DistTestCore +{ + public static class DownloadedLogExtensions + { + public static void AssertLogContains(this IDownloadedLog log, string expectedString) + { + Assert.That(log.DoesLogContain(expectedString), $"Did not find '{expectedString}' in log."); + } + } +} diff --git a/Tests/AutoBootstrapDistTest.cs b/Tests/AutoBootstrapDistTest.cs new file mode 100644 index 0000000..950b082 --- /dev/null +++ b/Tests/AutoBootstrapDistTest.cs @@ -0,0 +1,45 @@ +using CodexPlugin; +using DistTestCore; +using NUnit.Framework; + +namespace Tests +{ + public class AutoBootstrapDistTest : DistTest + { + public IOnlineCodexNode AddCodex() + { + return AddCodex(s => { }); + } + + public IOnlineCodexNode AddCodex(Action setup) + { + return this.SetupCodexNode(s => + { + setup(s); + s.WithBootstrapNode(BootstrapNode); + }); + } + + public ICodexNodeGroup AddCodex(int numberOfNodes) + { + return this.SetupCodexNodes(numberOfNodes, s => s.WithBootstrapNode(BootstrapNode)); + } + + public ICodexNodeGroup AddCodex(int numberOfNodes, Action setup) + { + return this.SetupCodexNodes(numberOfNodes, s => + { + setup(s); + s.WithBootstrapNode(BootstrapNode); + }); + } + + [SetUp] + public void SetUpBootstrapNode() + { + BootstrapNode = this.SetupCodexNode(s => s.WithName("BOOTSTRAP")); + } + + protected IOnlineCodexNode BootstrapNode { get; private set; } = null!; + } +} diff --git a/Tests/BasicTests/ContinuousSubstitute.cs b/Tests/BasicTests/ContinuousSubstitute.cs new file mode 100644 index 0000000..41b8da6 --- /dev/null +++ b/Tests/BasicTests/ContinuousSubstitute.cs @@ -0,0 +1,253 @@ +using CodexPlugin; +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace Tests.BasicTests +{ + [Ignore("Used for debugging continuous tests")] + [TestFixture] + public class ContinuousSubstitute : AutoBootstrapDistTest + { + [Test] + public void ContinuousTestSubstitute() + { + var group = AddCodex(5, o => o + //.EnableMetrics() + //.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) + .WithBlockTTL(TimeSpan.FromMinutes(2)) + .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2)) + .WithBlockMaintenanceNumber(10000) + .WithBlockTTL(TimeSpan.FromMinutes(2)) + .WithStorageQuota(1.GB())); + + var nodes = group.Cast().ToArray(); + + foreach (var node in nodes) + { + //node.Marketplace.MakeStorageAvailable( + //size: 500.MB(), + //minPricePerBytePerSecond: 1.TestTokens(), + //maxCollateral: 1024.TestTokens(), + //maxDuration: TimeSpan.FromMinutes(5)); + } + + var endTime = DateTime.UtcNow + TimeSpan.FromHours(10); + while (DateTime.UtcNow < endTime) + { + var allNodes = nodes.ToList(); + var primary = allNodes.PickOneRandom(); + var secondary = allNodes.PickOneRandom(); + + Log("Run Test"); + PerformTest(primary, secondary); + + Thread.Sleep(TimeSpan.FromSeconds(5)); + } + } + + [Test] + public void PeerTest() + { + var group = AddCodex(5, o => o + //.EnableMetrics() + //.EnableMarketplace(100000.TestTokens(), 0.Eth(), isValidator: true) + .WithBlockTTL(TimeSpan.FromMinutes(2)) + .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2)) + .WithBlockMaintenanceNumber(10000) + .WithBlockTTL(TimeSpan.FromMinutes(2)) + .WithStorageQuota(1.GB())); + + var nodes = group.Cast().ToArray(); + + var checkTime = DateTime.UtcNow + TimeSpan.FromMinutes(1); + var endTime = DateTime.UtcNow + TimeSpan.FromHours(10); + while (DateTime.UtcNow < endTime) + { + //CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes()); + //CheckRoutingTables(GetAllOnlineCodexNodes()); + + var node = RandomUtils.PickOneRandom(nodes.ToList()); + var file = GenerateTestFile(50.MB()); + node.UploadFile(file); + + Thread.Sleep(20000); + } + } + + private void CheckRoutingTables(IEnumerable nodes) + { + var all = nodes.ToArray(); + var allIds = all.Select(n => n.GetDebugInfo().table.localNode.nodeId).ToArray(); + + var errors = all.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray(); + + if (errors.Any()) + { + Assert.Fail(string.Join(Environment.NewLine, errors)); + } + } + + private string AreAllPresent(IOnlineCodexNode n, string[] allIds) + { + var info = n.GetDebugInfo(); + var known = info.table.nodes.Select(n => n.nodeId).ToArray(); + var expected = allIds.Where(i => i != info.table.localNode.nodeId).ToArray(); + + if (!expected.All(ex => known.Contains(ex))) + { + return $"Not all of '{string.Join(",", expected)}' were present in routing table: '{string.Join(",", known)}'"; + } + + return string.Empty; + } + + private ByteSize fileSize = 80.MB(); + + private void PerformTest(IOnlineCodexNode primary, IOnlineCodexNode secondary) + { + ScopedTestFiles(() => + { + var testFile = GenerateTestFile(fileSize); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = secondary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + }); + } + + [Test] + public void HoldMyBeerTest() + { + var blockExpirationTime = TimeSpan.FromMinutes(3); + var group = AddCodex(3, o => o + //.EnableMetrics() + .WithBlockTTL(blockExpirationTime) + .WithBlockMaintenanceInterval(TimeSpan.FromMinutes(2)) + .WithBlockMaintenanceNumber(10000) + .WithStorageQuota(2000.MB())); + + var nodes = group.Cast().ToArray(); + + var endTime = DateTime.UtcNow + TimeSpan.FromHours(24); + + var filesize = 80.MB(); + double codexDefaultBlockSize = 31 * 64 * 33; + var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize)); + var sizeInBytes = filesize.SizeInBytes; + Assert.That(numberOfBlocks, Is.EqualTo(1282)); + + var startTime = DateTime.UtcNow; + var successfulUploads = 0; + var successfulDownloads = 0; + + while (DateTime.UtcNow < endTime) + { + foreach (var node in nodes) + { + try + { + Thread.Sleep(TimeSpan.FromSeconds(5)); + + ScopedTestFiles(() => + { + var uploadStartTime = DateTime.UtcNow; + var file = GenerateTestFile(filesize); + var cid = node.UploadFile(file); + + var cidTag = cid.Id.Substring(cid.Id.Length - 6); + Measure("upload-log-asserts", () => + { + var uploadLog = node.DownloadLog(tailLines: 50000); + + var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\""); + uploadLog.DeleteFile(); + + var storeLine = GetLineForCidTag(storeLines, cidTag); + AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes); + }); + successfulUploads++; + + var uploadTimeTaken = DateTime.UtcNow - uploadStartTime; + if (uploadTimeTaken >= blockExpirationTime.Subtract(TimeSpan.FromSeconds(10))) + { + Assert.Fail("Upload took too long. Blocks already expired."); + } + + var dl = node.DownloadContent(cid); + file.AssertIsEqual(dl); + + Measure("download-log-asserts", () => + { + var downloadLog = node.DownloadLog(tailLines: 50000); + + var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\""); + downloadLog.DeleteFile(); + + var sentLine = GetLineForCidTag(sentLines, cidTag); + AssertSentLineContains(sentLine, sizeInBytes); + }); + successfulDownloads++; + }); + } + catch + { + var testDuration = DateTime.UtcNow - startTime; + Log("Test failed. Delaying shut-down by 30 seconds to collect metrics."); + Log($"Test failed after {Time.FormatDuration(testDuration)} and {successfulUploads} successful uploads and {successfulDownloads} successful downloads"); + Thread.Sleep(TimeSpan.FromSeconds(30)); + throw; + } + } + + Thread.Sleep(TimeSpan.FromSeconds(5)); + } + } + + private void AssertSentLineContains(string sentLine, long sizeInBytes) + { + var tag = "bytes="; + var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length); + var bytes = Convert.ToInt64(token); + Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'"); + } + + private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes) + { + var tokens = storeLine.Split(" "); + + var blocksToken = GetToken(tokens, "blocks="); + var sizeToken = GetToken(tokens, "size="); + if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine); + if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine); + + var blocks = Convert.ToInt64(blocksToken); + var size = Convert.ToInt64(sizeToken?.Replace("'NByte", "")); + + var lineLog = $" Line: '{storeLine}'"; + Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog); + Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog); + } + + private string GetLineForCidTag(string[] lines, string cidTag) + { + var result = lines.SingleOrDefault(l => l.Contains(cidTag)); + if (result == null) + { + Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'"); + throw new Exception(); + } + + return result; + } + + private string? GetToken(string[] tokens, string tag) + { + var token = tokens.SingleOrDefault(t => t.StartsWith(tag)); + if (token == null) return null; + return token.Substring(tag.Length); + } + } +} diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs new file mode 100644 index 0000000..8934386 --- /dev/null +++ b/Tests/BasicTests/ExampleTests.cs @@ -0,0 +1,87 @@ +using CodexPlugin; +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace Tests.BasicTests +{ + [TestFixture] + public class ExampleTests : DistTest + { + [Test] + public void CodexLogExample() + { + var primary = this.SetupCodexNode(); + + primary.UploadFile(GenerateTestFile(5.MB())); + + var log = primary.DownloadLog(); + + log.AssertLogContains("Uploaded file"); + } + + [Test] + public void TwoMetricsExample() + { + //var group = this.SetupCodexNodes(2, s => s.EnableMetrics()); + //var group2 = this.SetupCodexNodes(2, s => s.EnableMetrics()); + + //var primary = group[0]; + //var secondary = group[1]; + //var primary2 = group2[0]; + //var secondary2 = group2[1]; + + //primary.ConnectToPeer(secondary); + //primary2.ConnectToPeer(secondary2); + + //Thread.Sleep(TimeSpan.FromMinutes(2)); + + //primary.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + //primary2.Metrics.AssertThat("libp2p_peers", Is.EqualTo(1)); + } + + [Test] + public void MarketplaceExample() + { + //var sellerInitialBalance = 234.TestTokens(); + //var buyerInitialBalance = 1000.TestTokens(); + //var fileSize = 10.MB(); + + //var seller = this.SetupCodexNode(s => s + // .WithStorageQuota(11.GB()) + // .EnableMarketplace(sellerInitialBalance)); + + //seller.Marketplace.AssertThatBalance(Is.EqualTo(sellerInitialBalance)); + //seller.Marketplace.MakeStorageAvailable( + // size: 10.GB(), + // minPricePerBytePerSecond: 1.TestTokens(), + // maxCollateral: 20.TestTokens(), + // maxDuration: TimeSpan.FromMinutes(3)); + + //var testFile = GenerateTestFile(fileSize); + + //var buyer = this.SetupCodexNode(s => s + // .WithBootstrapNode(seller) + // .EnableMarketplace(buyerInitialBalance)); + + //buyer.Marketplace.AssertThatBalance(Is.EqualTo(buyerInitialBalance)); + + //var contentId = buyer.UploadFile(testFile); + //var purchaseContract = buyer.Marketplace.RequestStorage(contentId, + // pricePerSlotPerSecond: 2.TestTokens(), + // requiredCollateral: 10.TestTokens(), + // minRequiredNumberOfNodes: 1, + // proofProbability: 5, + // duration: TimeSpan.FromMinutes(1)); + + //purchaseContract.WaitForStorageContractStarted(fileSize); + + //seller.Marketplace.AssertThatBalance(Is.LessThan(sellerInitialBalance), "Collateral was not placed."); + + //purchaseContract.WaitForStorageContractFinished(); + + //seller.Marketplace.AssertThatBalance(Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage."); + //buyer.Marketplace.AssertThatBalance(Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage."); + } + } +} diff --git a/Tests/BasicTests/NetworkIsolationTest.cs b/Tests/BasicTests/NetworkIsolationTest.cs new file mode 100644 index 0000000..da1be63 --- /dev/null +++ b/Tests/BasicTests/NetworkIsolationTest.cs @@ -0,0 +1,47 @@ +using CodexPlugin; +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace Tests.BasicTests +{ + // Warning! + // This is a test to check network-isolation in the test-infrastructure. + // It requires parallelism(2) or greater to run. + [TestFixture] + [Ignore("Disabled until a solution is implemented.")] + public class NetworkIsolationTest : DistTest + { + private IOnlineCodexNode? node = null; + + [Test] + public void SetUpANodeAndWait() + { + node = this.SetupCodexNode(); + + Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5)); + } + + [Test] + public void ForeignNodeConnects() + { + var myNode = this.SetupCodexNode(); + + Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5)); + + try + { + myNode.ConnectToPeer(node!); + } + catch + { + // Good! This connection should be prohibited by the network isolation policy. + node = null; + return; + } + + Assert.Fail("Connection could be established between two Codex nodes running in different namespaces. " + + "This may cause cross-test interference. Network isolation policy should be applied. Test infra failure."); + } + } +} diff --git a/Tests/BasicTests/OneClientTests.cs b/Tests/BasicTests/OneClientTests.cs new file mode 100644 index 0000000..5fa2636 --- /dev/null +++ b/Tests/BasicTests/OneClientTests.cs @@ -0,0 +1,42 @@ +using CodexPlugin; +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace Tests.BasicTests +{ + [TestFixture] + public class OneClientTests : DistTest + { + [Test] + public void OneClientTest() + { + var primary = this.SetupCodexNode(); + + PerformOneClientTest(primary); + } + + [Test] + public void RestartTest() + { + var primary = this.SetupCodexNode(); + + //var setup = primary.BringOffline(); + + //primary = BringOnline(setup)[0]; + + PerformOneClientTest(primary); + } + + private void PerformOneClientTest(IOnlineCodexNode primary) + { + var testFile = GenerateTestFile(1.MB()); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = primary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + } +} diff --git a/Tests/BasicTests/ThreeClientTest.cs b/Tests/BasicTests/ThreeClientTest.cs new file mode 100644 index 0000000..5e44c97 --- /dev/null +++ b/Tests/BasicTests/ThreeClientTest.cs @@ -0,0 +1,25 @@ +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace Tests.BasicTests +{ + [TestFixture] + public class ThreeClientTest : AutoBootstrapDistTest + { + [Test] + public void ThreeClient() + { + var primary = AddCodex(); + var secondary = AddCodex(); + + var testFile = GenerateTestFile(10.MB()); + + var contentId = primary.UploadFile(testFile); + + var downloadedFile = secondary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + } +} diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs index ed16790..909f29c 100644 --- a/Tests/BasicTests/TwoClientTests.cs +++ b/Tests/BasicTests/TwoClientTests.cs @@ -1,8 +1,8 @@ -using DistTestCore; +using CodexPlugin; +using DistTestCore; using KubernetesWorkflow; using NUnit.Framework; using Utils; -using CodexPlugin; namespace Tests.BasicTests { diff --git a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs new file mode 100644 index 0000000..4728389 --- /dev/null +++ b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -0,0 +1,43 @@ +using CodexPlugin; +using DistTestCore; +using NUnit.Framework; +using Utils; + +namespace Tests.DownloadConnectivityTests +{ + [TestFixture] + public class FullyConnectedDownloadTests : AutoBootstrapDistTest + { + [Test] + public void MetricsDoesNotInterfereWithPeerDownload() + { + //AddCodex(2, s => s.EnableMetrics()); + + AssertAllNodesConnected(); + } + + [Test] + public void MarketplaceDoesNotInterfereWithPeerDownload() + { + //AddCodex(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens())); + + AssertAllNodesConnected(); + } + + [Test] + [Combinatorial] + public void FullyConnectedDownloadTest( + [Values(3, 5)] int numberOfNodes, + [Values(10, 80)] int sizeMBs) + { + AddCodex(numberOfNodes); + + AssertAllNodesConnected(sizeMBs); + } + + private void AssertAllNodesConnected(int sizeMBs = 10) + { + //CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); + } + } +} diff --git a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs new file mode 100644 index 0000000..aee601b --- /dev/null +++ b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -0,0 +1,53 @@ +using CodexPlugin; +using DistTestCore; +using NUnit.Framework; + +namespace Tests.PeerDiscoveryTests +{ + [TestFixture] + public class LayeredDiscoveryTests : DistTest + { + [Test] + public void TwoLayersTest() + { + var root = this.SetupCodexNode(); + var l1Source = this.SetupCodexNode(s => s.WithBootstrapNode(root)); + var l1Node = this.SetupCodexNode(s => s.WithBootstrapNode(root)); + var l2Target = this.SetupCodexNode(s => s.WithBootstrapNode(l1Node)); + + AssertAllNodesConnected(); + } + + [Test] + public void ThreeLayersTest() + { + var root = this.SetupCodexNode(); + var l1Source = this.SetupCodexNode(s => s.WithBootstrapNode(root)); + var l1Node = this.SetupCodexNode(s => s.WithBootstrapNode(root)); + var l2Node = this.SetupCodexNode(s => s.WithBootstrapNode(l1Node)); + var l3Target = this.SetupCodexNode(s => s.WithBootstrapNode(l2Node)); + + AssertAllNodesConnected(); + } + + [TestCase(3)] + [TestCase(5)] + [TestCase(10)] + [TestCase(20)] + public void NodeChainTest(int chainLength) + { + var node = this.SetupCodexNode(); + for (var i = 1; i < chainLength; i++) + { + node = this.SetupCodexNode(s => s.WithBootstrapNode(node)); + } + + AssertAllNodesConnected(); + } + + private void AssertAllNodesConnected() + { + //CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes()); + } + } +} diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs new file mode 100644 index 0000000..350afc7 --- /dev/null +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -0,0 +1,51 @@ +using DistTestCore; +using NUnit.Framework; + +namespace Tests.PeerDiscoveryTests +{ + [TestFixture] + public class PeerDiscoveryTests : AutoBootstrapDistTest + { + [Test] + public void CanReportUnknownPeerId() + { + var unknownId = "16Uiu2HAkv2CHWpff3dj5iuVNERAp8AGKGNgpGjPexJZHSqUstfsK"; + var node = AddCodex(); + + var result = node.GetDebugPeer(unknownId); + Assert.That(result.IsPeerFound, Is.False); + } + + [Test] + public void MetricsDoesNotInterfereWithPeerDiscovery() + { + //AddCodex(2, s => s.EnableMetrics()); + + AssertAllNodesConnected(); + } + + [Test] + public void MarketplaceDoesNotInterfereWithPeerDiscovery() + { + //AddCodex(2, s => s.EnableMarketplace(1000.TestTokens())); + + AssertAllNodesConnected(); + } + + [TestCase(2)] + [TestCase(3)] + [TestCase(10)] + [TestCase(20)] + public void VariableNodes(int number) + { + AddCodex(number); + + AssertAllNodesConnected(); + } + + private void AssertAllNodesConnected() + { + //CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes()); + } + } +}