From 7eda26d17757b1d740462384053c15e79cf5a294 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 24 Aug 2023 10:59:11 +0200 Subject: [PATCH 1/9] Cleanup of full-connectivity helper. --- DistTestCore/DistTest.cs | 16 +++++++++----- .../Helpers/FullConnectivityHelper.cs | 22 ++++++++++++------- .../Helpers/PeerConnectionTestHelpers.cs | 5 +++-- .../Helpers/PeerDownloadTestHelpers.cs | 18 ++++++++++----- .../LongFullyConnectedDownloadTests.cs | 5 ++--- .../FullyConnectedDownloadTests.cs | 2 +- .../LayeredDiscoveryTests.cs | 3 +-- .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 3 +-- 8 files changed, 44 insertions(+), 30 deletions(-) diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index 943b098..a6379ec 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -30,14 +30,8 @@ namespace DistTestCore var startTime = DateTime.UtcNow; fixtureLog = new FixtureLog(logConfig, startTime); statusLog = new StatusLog(logConfig, startTime); - - PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this); - PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this); } - public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; } - public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; } - [OneTimeSetUp] public void GlobalSetup() { @@ -175,6 +169,16 @@ namespace DistTestCore GetTestLog().Debug(msg); } + public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers() + { + return new PeerConnectionTestHelpers(GetTestLog()); + } + + public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers() + { + return new PeerDownloadTestHelpers(GetTestLog(), Get().FileManager); + } + public void Measure(string name, Action action) { Stopwatch.Measure(Get().Log, name, action); diff --git a/DistTestCore/Helpers/FullConnectivityHelper.cs b/DistTestCore/Helpers/FullConnectivityHelper.cs index afa3442..0ac6f82 100644 --- a/DistTestCore/Helpers/FullConnectivityHelper.cs +++ b/DistTestCore/Helpers/FullConnectivityHelper.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using Logging; using NUnit.Framework; using Utils; @@ -14,12 +15,12 @@ namespace DistTestCore.Helpers public class FullConnectivityHelper { private static string Nl = Environment.NewLine; - private readonly DistTest test; + private readonly BaseLog log; private readonly IFullConnectivityImplementation implementation; - public FullConnectivityHelper(DistTest test, IFullConnectivityImplementation implementation) + public FullConnectivityHelper(BaseLog log, IFullConnectivityImplementation implementation) { - this.test = test; + this.log = log; this.implementation = implementation; } @@ -30,7 +31,7 @@ namespace DistTestCore.Helpers private void AssertFullyConnected(IOnlineCodexNode[] nodes) { - test.Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); + Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); var entries = CreateEntries(nodes); var pairs = CreatePairs(entries); @@ -43,13 +44,13 @@ namespace DistTestCore.Helpers { var pairDetails = string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages())); - test.Log($"Connections failed:{Nl}{pairDetails}"); + Log($"Connections failed:{Nl}{pairDetails}"); Assert.Fail(string.Join(Nl, pairs.SelectMany(p => p.GetResultMessages()))); } else { - test.Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}"); + Log($"'{implementation.Description()}' = Success! for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}"); } } @@ -72,7 +73,7 @@ namespace DistTestCore.Helpers foreach (var pair in selectedPair) { - test.ScopedTestFiles(pair.Check); + pair.Check(); if (pair.Success) { @@ -81,7 +82,7 @@ namespace DistTestCore.Helpers } } - test.Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}"); + Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}"); } private Entry[] CreateEntries(IOnlineCodexNode[] nodes) @@ -117,6 +118,11 @@ namespace DistTestCore.Helpers } } + private void Log(string msg) + { + log.Log(msg); + } + public class Entry { public Entry(IOnlineCodexNode node) diff --git a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs index fa07b06..6b1a673 100644 --- a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs +++ b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using Logging; using static DistTestCore.Helpers.FullConnectivityHelper; namespace DistTestCore.Helpers @@ -7,9 +8,9 @@ namespace DistTestCore.Helpers { private readonly FullConnectivityHelper helper; - public PeerConnectionTestHelpers(DistTest test) + public PeerConnectionTestHelpers(BaseLog log) { - helper = new FullConnectivityHelper(test, this); + helper = new FullConnectivityHelper(log, this); } public void AssertFullyConnected(IEnumerable nodes) diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs index ffb7d52..4ba22a5 100644 --- a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -1,18 +1,19 @@ -using static DistTestCore.Helpers.FullConnectivityHelper; +using Logging; +using static DistTestCore.Helpers.FullConnectivityHelper; namespace DistTestCore.Helpers { public class PeerDownloadTestHelpers : IFullConnectivityImplementation { private readonly FullConnectivityHelper helper; - private readonly DistTest test; + private readonly FileManager fileManager; private ByteSize testFileSize; - public PeerDownloadTestHelpers(DistTest test) + public PeerDownloadTestHelpers(BaseLog log, FileManager fileManager) { - helper = new FullConnectivityHelper(test, this); + helper = new FullConnectivityHelper(log, this); testFileSize = 1.MB(); - this.test = test; + this.fileManager = fileManager; } public void AssertFullDownloadInterconnectivity(IEnumerable nodes, ByteSize testFileSize) @@ -33,6 +34,7 @@ namespace DistTestCore.Helpers public PeerConnectionState Check(Entry from, Entry to) { + fileManager.PushFileSet(); var expectedFile = GenerateTestFile(from.Node, to.Node); var contentId = from.Node.UploadFile(expectedFile); @@ -49,6 +51,10 @@ namespace DistTestCore.Helpers // We consider that as no-connection for the purpose of this test. return PeerConnectionState.NoConnection; } + finally + { + fileManager.PopFileSet(); + } // Should an exception occur during upload, then this try is inconclusive and we try again next loop. } @@ -58,7 +64,7 @@ namespace DistTestCore.Helpers var up = uploader.GetName().Replace("<", "").Replace(">", ""); var down = downloader.GetName().Replace("<", "").Replace(">", ""); var label = $"~from:{up}-to:{down}~"; - return test.GenerateTestFile(testFileSize, label); + return fileManager.GenerateTestFile(testFileSize, label); } } } diff --git a/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs b/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs index 5bb7323..d0650f6 100644 --- a/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs +++ b/LongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs @@ -1,5 +1,4 @@ -using DistTestCore.Helpers; -using DistTestCore; +using DistTestCore; using NUnit.Framework; namespace TestsLong.DownloadConnectivityTests @@ -16,7 +15,7 @@ namespace TestsLong.DownloadConnectivityTests { for (var i = 0; i < numberOfNodes; i++) SetupCodexNode(); - PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); + CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); } } } diff --git a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index a1f2c5f..4f91f64 100644 --- a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -14,7 +14,7 @@ namespace Tests.DownloadConnectivityTests { for (var i = 0; i < numberOfNodes; i++) SetupCodexNode(); - PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); + CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); } } } diff --git a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index aeb414a..8cd32a2 100644 --- a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -1,5 +1,4 @@ using DistTestCore; -using DistTestCore.Helpers; using NUnit.Framework; namespace Tests.PeerDiscoveryTests @@ -47,7 +46,7 @@ namespace Tests.PeerDiscoveryTests private void AssertAllNodesConnected() { - PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); + CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes()); } } } diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 002fe79..94e7075 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -1,5 +1,4 @@ using DistTestCore; -using DistTestCore.Helpers; using NUnit.Framework; namespace Tests.PeerDiscoveryTests @@ -33,7 +32,7 @@ namespace Tests.PeerDiscoveryTests private void AssertAllNodesConnected() { - PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); + CreatePeerConnectionTestHelpers().AssertFullyConnected(GetAllOnlineCodexNodes()); } } } From eec70a9b8c42b63bc7a5e576fae2fe9a7a38f278 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 24 Aug 2023 11:32:32 +0200 Subject: [PATCH 2/9] Adds peer connectivity check after deployment --- CodexNetDeployer/CodexNodeStarter.cs | 16 +++++++-- CodexNetDeployer/Configuration.cs | 3 ++ CodexNetDeployer/Deployer.cs | 29 +++++++++++----- CodexNetDeployer/PeerConnectivityChecker.cs | 34 +++++++++++++++++++ DistTestCore/Codex/CodexAccess.cs | 5 +++ .../Helpers/FullConnectivityHelper.cs | 10 +++--- .../Helpers/PeerConnectionTestHelpers.cs | 10 ++++-- .../Helpers/PeerDownloadTestHelpers.cs | 19 ++++++++--- 8 files changed, 103 insertions(+), 23 deletions(-) create mode 100644 CodexNetDeployer/PeerConnectivityChecker.cs diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 39bc1f9..79435c9 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -21,7 +21,7 @@ namespace CodexNetDeployer validatorsLeft = numberOfValidators; } - public RunningContainer? Start(int i) + public CodexNodeStartResult? Start(int i) { Console.Write($" - {i} = "); var workflow = lifecycle.WorkflowCreator.CreateWorkflow(); @@ -60,7 +60,7 @@ namespace CodexNetDeployer if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; validatorsLeft--; - return container; + return new CodexNodeStartResult(container, codexAccess); } } } @@ -102,4 +102,16 @@ namespace CodexNetDeployer return codexStart; } } + + public class CodexNodeStartResult + { + public CodexNodeStartResult(RunningContainer container, CodexAccess access) + { + Container = container; + Access = access; + } + + public RunningContainer Container { get; } + public CodexAccess Access { get; } + } } diff --git a/CodexNetDeployer/Configuration.cs b/CodexNetDeployer/Configuration.cs index 4df18be..7ad241a 100644 --- a/CodexNetDeployer/Configuration.cs +++ b/CodexNetDeployer/Configuration.cs @@ -57,6 +57,9 @@ namespace CodexNetDeployer [Uniform("teststype-podlabel", "ttpl", "TESTSTYPE-PODLABEL", false, "Each kubernetes pod will be created with a label 'teststype' with value 'continuous'. " + "set this option to override the label value.")] public string TestsTypePodLabel { get; set; } = "continuous-tests"; + + [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment.")] + public bool CheckPeerConnection { get; set; } = false; public List Validate() { diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index 55ad4fe..f589813 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -8,14 +8,14 @@ namespace CodexNetDeployer public class Deployer { private readonly Configuration config; - private readonly NullLog log; private readonly DefaultTimeSet timeset; + private readonly PeerConnectivityChecker peerConnectivityChecker; public Deployer(Configuration config) { this.config = config; - log = new NullLog(); timeset = new DefaultTimeSet(); + peerConnectivityChecker = new PeerConnectivityChecker(); } public CodexDeployment Deploy() @@ -45,16 +45,18 @@ namespace CodexNetDeployer // Each node must have its own IP, so it needs it own pod. Start them 1 at a time. var codexStarter = new CodexNodeStarter(config, lifecycle, gethResults, config.NumberOfValidators!.Value); - var codexContainers = new List(); + var startResults = new List(); for (var i = 0; i < config.NumberOfCodexNodes; i++) { - var container = codexStarter.Start(i); - if (container != null) codexContainers.Add(container); + var result = codexStarter.Start(i); + if (result != null) startResults.Add(result); } - var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, codexContainers); + var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container)); - return new CodexDeployment(gethResults, codexContainers.ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata()); + CheckPeerConnectivity(startResults); + + return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata()); } private TestLifecycle CreateTestLifecycle() @@ -71,10 +73,10 @@ namespace CodexNetDeployer k8sNamespacePrefix: config.KubeNamespace ); - return new TestLifecycle(log, lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty); + return new TestLifecycle(new NullLog(), lifecycleConfig, timeset, config.TestsTypePodLabel, string.Empty); } - private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, List codexContainers) + private (RunningContainer?, GrafanaStartInfo?) StartMetricsService(TestLifecycle lifecycle, CodexSetup setup, IEnumerable codexContainers) { if (setup.MetricsMode == DistTestCore.Metrics.MetricsMode.None) return (null, null); @@ -95,6 +97,15 @@ namespace CodexNetDeployer return kubeConfigFile; } + private void CheckPeerConnectivity(List codexContainers) + { + if (!config.CheckPeerConnection) return; + + Log("Starting peer-connectivity check for deployed nodes..."); + peerConnectivityChecker.CheckConnectivity(codexContainers); + Log("Check passed."); + } + private DeploymentMetadata CreateMetadata() { return new DeploymentMetadata( diff --git a/CodexNetDeployer/PeerConnectivityChecker.cs b/CodexNetDeployer/PeerConnectivityChecker.cs new file mode 100644 index 0000000..a3baa17 --- /dev/null +++ b/CodexNetDeployer/PeerConnectivityChecker.cs @@ -0,0 +1,34 @@ +using DistTestCore.Helpers; +using Logging; + +namespace CodexNetDeployer +{ + public class PeerConnectivityChecker + { + public void CheckConnectivity(List startResults) + { + var checker = new PeerConnectionTestHelpers(new ConsoleLogger()); + + var access = startResults.Select(r => r.Access); + + checker.AssertFullyConnected(access); + } + } + + public class ConsoleLogger : BaseLog + { + public ConsoleLogger() : base(false) + { + } + + protected override string GetFullName() + { + return "CONSOLE"; + } + + public override void Log(string message) + { + Console.WriteLine(message); + } + } +} diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 076a335..57f7689 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -88,6 +88,11 @@ namespace DistTestCore.Codex return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); } + public string GetName() + { + return Container.Name; + } + private Http Http() { CheckContainerCrashed(); diff --git a/DistTestCore/Helpers/FullConnectivityHelper.cs b/DistTestCore/Helpers/FullConnectivityHelper.cs index 0ac6f82..228beeb 100644 --- a/DistTestCore/Helpers/FullConnectivityHelper.cs +++ b/DistTestCore/Helpers/FullConnectivityHelper.cs @@ -24,12 +24,12 @@ namespace DistTestCore.Helpers this.implementation = implementation; } - public void AssertFullyConnected(IEnumerable nodes) + public void AssertFullyConnected(IEnumerable nodes) { AssertFullyConnected(nodes.ToArray()); } - private void AssertFullyConnected(IOnlineCodexNode[] nodes) + private void AssertFullyConnected(CodexAccess[] nodes) { Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); var entries = CreateEntries(nodes); @@ -85,7 +85,7 @@ namespace DistTestCore.Helpers Log($"Connections successful:{Nl}{string.Join(Nl, pairDetails)}"); } - private Entry[] CreateEntries(IOnlineCodexNode[] nodes) + private Entry[] CreateEntries(CodexAccess[] nodes) { var entries = nodes.Select(n => new Entry(n)).ToArray(); @@ -125,13 +125,13 @@ namespace DistTestCore.Helpers public class Entry { - public Entry(IOnlineCodexNode node) + public Entry(CodexAccess node) { Node = node; Response = node.GetDebugInfo(); } - public IOnlineCodexNode Node { get; } + public CodexAccess Node { get; } public CodexDebugResponse Response { get; } public override string ToString() diff --git a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs index 6b1a673..65bb4ce 100644 --- a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs +++ b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs @@ -14,6 +14,11 @@ namespace DistTestCore.Helpers } public void AssertFullyConnected(IEnumerable nodes) + { + AssertFullyConnected(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess)); + } + + public void AssertFullyConnected(IEnumerable nodes) { helper.AssertFullyConnected(nodes); } @@ -58,9 +63,8 @@ namespace DistTestCore.Helpers var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId); if (peer == null) return $"peerId: {node.peerId} is not known."; - var n = (OnlineCodexNode)peer.Node; - var ip = n.CodexAccess.Container.Pod.PodInfo.Ip; - var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag); + var ip = peer.Node.Container.Pod.PodInfo.Ip; + var discPort = peer.Node.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag); return $"{ip}:{discPort.Number}"; } } diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs index 4ba22a5..b43b7ab 100644 --- a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -1,4 +1,5 @@ -using Logging; +using DistTestCore.Codex; +using Logging; using static DistTestCore.Helpers.FullConnectivityHelper; namespace DistTestCore.Helpers @@ -17,6 +18,11 @@ namespace DistTestCore.Helpers } public void AssertFullDownloadInterconnectivity(IEnumerable nodes, ByteSize testFileSize) + { + AssertFullDownloadInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess), testFileSize); + } + + public void AssertFullDownloadInterconnectivity(IEnumerable nodes, ByteSize testFileSize) { this.testFileSize = testFileSize; helper.AssertFullyConnected(nodes); @@ -37,11 +43,16 @@ namespace DistTestCore.Helpers fileManager.PushFileSet(); var expectedFile = GenerateTestFile(from.Node, to.Node); - var contentId = from.Node.UploadFile(expectedFile); + using var uploadStream = File.OpenRead(expectedFile.Filename); + var contentId = from.Node.UploadFile(uploadStream); try { - var downloadedFile = to.Node.DownloadContent(contentId, expectedFile.Label + "_downloaded"); + var downloadedFile = fileManager.CreateEmptyTestFile(expectedFile.Label + "_downloaded"); + using var downloadStream = File.OpenWrite(downloadedFile.Filename); + using var stream = to.Node.DownloadFile(contentId); + stream.CopyTo(downloadStream); + expectedFile.AssertIsEqual(downloadedFile); return PeerConnectionState.Connection; } @@ -59,7 +70,7 @@ namespace DistTestCore.Helpers // Should an exception occur during upload, then this try is inconclusive and we try again next loop. } - private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader) + private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader) { var up = uploader.GetName().Replace("<", "").Replace(">", ""); var down = downloader.GetName().Replace("<", "").Replace(">", ""); From 7da90fc53b5fe292e57061ae90418b36c77f7aa6 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 24 Aug 2023 15:32:08 +0200 Subject: [PATCH 3/9] cleanup --- CodexNetDeployer/PeerConnectivityChecker.cs | 8 ++++---- CodexNetDeployer/Program.cs | 3 --- .../FullyConnectedDownloadTests.cs | 2 +- Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs | 5 +---- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/CodexNetDeployer/PeerConnectivityChecker.cs b/CodexNetDeployer/PeerConnectivityChecker.cs index a3baa17..6e7eddb 100644 --- a/CodexNetDeployer/PeerConnectivityChecker.cs +++ b/CodexNetDeployer/PeerConnectivityChecker.cs @@ -7,17 +7,17 @@ namespace CodexNetDeployer { public void CheckConnectivity(List startResults) { - var checker = new PeerConnectionTestHelpers(new ConsoleLogger()); - + var log = new ConsoleLog(); + var checker = new PeerConnectionTestHelpers(log); var access = startResults.Select(r => r.Access); checker.AssertFullyConnected(access); } } - public class ConsoleLogger : BaseLog + public class ConsoleLog : BaseLog { - public ConsoleLogger() : base(false) + public ConsoleLog() : base(false) { } diff --git a/CodexNetDeployer/Program.cs b/CodexNetDeployer/Program.cs index da18e01..8ddf527 100644 --- a/CodexNetDeployer/Program.cs +++ b/CodexNetDeployer/Program.cs @@ -56,8 +56,5 @@ public class Program Console.WriteLine("CodexNetDeployer allows you to easily deploy multiple Codex nodes in a Kubernetes cluster. " + "The deployer will set up the required supporting services, deploy the Codex on-chain contracts, start and bootstrap the Codex instances. " + "All Kubernetes objects will be created in the namespace provided, allowing you to easily find, modify, and delete them afterwards." + nl); - - Console.WriteLine("CodexNetDeployer assumes you are running this tool from *inside* the Kubernetes cluster you want to deploy to. " + - "If you are not running this from a container inside the cluster, add the argument '--external'." + nl); } } diff --git a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index 4f91f64..a234aeb 100644 --- a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -12,7 +12,7 @@ namespace Tests.DownloadConnectivityTests [Values(1, 3, 5)] int numberOfNodes, [Values(1, 10)] int sizeMBs) { - for (var i = 0; i < numberOfNodes; i++) SetupCodexNode(); + SetupCodexNodes(numberOfNodes); CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); } diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 94e7075..170f850 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -22,10 +22,7 @@ namespace Tests.PeerDiscoveryTests [TestCase(20)] public void VariableNodesInPods(int number) { - for (var i = 0; i < number; i++) - { - SetupCodexNode(); - } + SetupCodexNodes(number); AssertAllNodesConnected(); } From b276592daf45e0e47a3facf7f7179cefbcec5c26 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 24 Aug 2023 15:56:56 +0200 Subject: [PATCH 4/9] Fixes download stream error in peerDownloadHelper --- DistTestCore/Helpers/PeerDownloadTestHelpers.cs | 15 ++++++++++----- Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs index b43b7ab..40024e1 100644 --- a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -48,11 +48,7 @@ namespace DistTestCore.Helpers try { - var downloadedFile = fileManager.CreateEmptyTestFile(expectedFile.Label + "_downloaded"); - using var downloadStream = File.OpenWrite(downloadedFile.Filename); - using var stream = to.Node.DownloadFile(contentId); - stream.CopyTo(downloadStream); - + var downloadedFile = DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded"); expectedFile.AssertIsEqual(downloadedFile); return PeerConnectionState.Connection; } @@ -70,6 +66,15 @@ namespace DistTestCore.Helpers // Should an exception occur during upload, then this try is inconclusive and we try again next loop. } + private TestFile DownloadFile(CodexAccess node, string contentId, string label) + { + var downloadedFile = fileManager.CreateEmptyTestFile(label); + using var downloadStream = File.OpenWrite(downloadedFile.Filename); + using var stream = node.DownloadFile(contentId); + stream.CopyTo(downloadStream); + return downloadedFile; + } + private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader) { var up = uploader.GetName().Replace("<", "").Replace(">", ""); diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 170f850..2ca67e6 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -20,7 +20,7 @@ namespace Tests.PeerDiscoveryTests [TestCase(3)] [TestCase(10)] [TestCase(20)] - public void VariableNodesInPods(int number) + public void VariableNodes(int number) { SetupCodexNodes(number); From c22ba346af5c197b8b6d94a4aded6a68953cd23a Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 25 Aug 2023 08:32:08 +0200 Subject: [PATCH 5/9] Adds peer info to dashboard --- DistTestCore/Metrics/dashboard.json | 197 ++++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/DistTestCore/Metrics/dashboard.json b/DistTestCore/Metrics/dashboard.json index 3af1c52..313abf0 100644 --- a/DistTestCore/Metrics/dashboard.json +++ b/DistTestCore/Metrics/dashboard.json @@ -512,6 +512,203 @@ "title": "Bytes used", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 24, + "panels": [], + "title": "Network Status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "libp2p_peers", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Peers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "c89eaad3-9184-429f-ac94-8ba0b1824dbb" + }, + "editorMode": "builder", + "expr": "routing_table_nodes", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Routing table nodes", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { From 249b13d50332fd3d6dc9fae8210023f80ec72ebd Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 28 Aug 2023 11:16:58 +0200 Subject: [PATCH 6/9] Adds tests to ensure marketplace and metrics don't interfere with peer discovery --- CodexNetDeployer/deploy-continuous-testnet.sh | 3 ++- Tests/BasicTests/TwoClientTests.cs | 11 +--------- .../FullyConnectedDownloadTests.cs | 21 +++++++++++++++++++ .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 16 ++++++++++++++ 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/CodexNetDeployer/deploy-continuous-testnet.sh b/CodexNetDeployer/deploy-continuous-testnet.sh index 25a67c7..afcfc44 100644 --- a/CodexNetDeployer/deploy-continuous-testnet.sh +++ b/CodexNetDeployer/deploy-continuous-testnet.sh @@ -12,4 +12,5 @@ dotnet run \ --block-ttl=180 \ --block-mi=120 \ --block-mn=10000 \ - --metrics=Dashboard + --metrics=Dashboard \ + --check-connect=1 diff --git a/Tests/BasicTests/TwoClientTests.cs b/Tests/BasicTests/TwoClientTests.cs index d03633b..12bab13 100644 --- a/Tests/BasicTests/TwoClientTests.cs +++ b/Tests/BasicTests/TwoClientTests.cs @@ -8,7 +8,7 @@ namespace Tests.BasicTests public class TwoClientTests : DistTest { [Test] - public void TwoClientsOnePodTest() + public void TwoClientTest() { var group = SetupCodexNodes(2); @@ -18,15 +18,6 @@ namespace Tests.BasicTests PerformTwoClientTest(primary, secondary); } - [Test] - public void TwoClientsTwoPodsTest() - { - var primary = SetupCodexNode(); - var secondary = SetupCodexNode(); - - PerformTwoClientTest(primary, secondary); - } - [Test] public void TwoClientsTwoLocationsTest() { diff --git a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index a234aeb..7cf546b 100644 --- a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -6,6 +6,22 @@ namespace Tests.DownloadConnectivityTests [TestFixture] public class FullyConnectedDownloadTests : AutoBootstrapDistTest { + [Test] + public void MetricsDoesNotInterfereWithPeerDownload() + { + SetupCodexNodes(2, s => s.EnableMetrics()); + + AssertAllNodesConnected(); + } + + [Test] + public void MarketplaceDoesNotInterfereWithPeerDownload() + { + SetupCodexNodes(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens())); + + AssertAllNodesConnected(); + } + [Test] [Combinatorial] public void FullyConnectedDownloadTest( @@ -14,6 +30,11 @@ namespace Tests.DownloadConnectivityTests { SetupCodexNodes(numberOfNodes); + AssertAllNodesConnected(sizeMBs); + } + + private void AssertAllNodesConnected(int sizeMBs = 10) + { CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes(), sizeMBs.MB()); } } diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 2ca67e6..fcb44fd 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -16,6 +16,22 @@ namespace Tests.PeerDiscoveryTests Assert.That(result.IsPeerFound, Is.False); } + [Test] + public void MetricsDoesNotInterfereWithPeerDiscovery() + { + SetupCodexNodes(2, s => s.EnableMetrics()); + + AssertAllNodesConnected(); + } + + [Test] + public void MarketplaceDoesNotInterfereWithPeerDiscovery() + { + SetupCodexNodes(2, s => s.EnableMarketplace(1000.TestTokens())); + + AssertAllNodesConnected(); + } + [TestCase(2)] [TestCase(3)] [TestCase(10)] From 553a368714e75074861044e5a9397a656bbb9c52 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 28 Aug 2023 11:53:59 +0200 Subject: [PATCH 7/9] Adds a container-restart check at the end of the deployment. --- CodexNetDeployer/CodexNodeStarter.cs | 6 ++++-- CodexNetDeployer/Deployer.cs | 21 +++++++++++++++++++++ KubernetesWorkflow/CrashWatcher.cs | 18 ++++++++++++++---- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/CodexNetDeployer/CodexNodeStarter.cs b/CodexNetDeployer/CodexNodeStarter.cs index 79435c9..1bc4af9 100644 --- a/CodexNetDeployer/CodexNodeStarter.cs +++ b/CodexNetDeployer/CodexNodeStarter.cs @@ -60,7 +60,7 @@ namespace CodexNetDeployer if (string.IsNullOrEmpty(bootstrapSpr)) bootstrapSpr = debugInfo.spr; validatorsLeft--; - return new CodexNodeStartResult(container, codexAccess); + return new CodexNodeStartResult(workflow, container, codexAccess); } } } @@ -105,12 +105,14 @@ namespace CodexNetDeployer public class CodexNodeStartResult { - public CodexNodeStartResult(RunningContainer container, CodexAccess access) + public CodexNodeStartResult(StartupWorkflow workflow, RunningContainer container, CodexAccess access) { + Workflow = workflow; Container = container; Access = access; } + public StartupWorkflow Workflow { get; } public RunningContainer Container { get; } public CodexAccess Access { get; } } diff --git a/CodexNetDeployer/Deployer.cs b/CodexNetDeployer/Deployer.cs index f589813..feaff51 100644 --- a/CodexNetDeployer/Deployer.cs +++ b/CodexNetDeployer/Deployer.cs @@ -55,6 +55,7 @@ namespace CodexNetDeployer var (prometheusContainer, grafanaStartInfo) = StartMetricsService(lifecycle, setup, startResults.Select(r => r.Container)); CheckPeerConnectivity(startResults); + CheckContainerRestarts(startResults); return new CodexDeployment(gethResults, startResults.Select(r => r.Container).ToArray(), prometheusContainer, grafanaStartInfo, CreateMetadata()); } @@ -106,6 +107,26 @@ namespace CodexNetDeployer Log("Check passed."); } + private void CheckContainerRestarts(List startResults) + { + var crashes = new List(); + foreach (var startResult in startResults) + { + var watcher = startResult.Workflow.CreateCrashWatcher(startResult.Container); + if (watcher.HasContainerCrashed()) crashes.Add(startResult.Container); + } + + if (!crashes.Any()) + { + Log("Container restart check passed."); + } + else + { + Log($"Deployment failed. The following containers have crashed: {string.Join(",", crashes.Select(c => c.Name))}"); + throw new Exception("Deployment failed: One or more containers crashed."); + } + } + private DeploymentMetadata CreateMetadata() { return new DeploymentMetadata( diff --git a/KubernetesWorkflow/CrashWatcher.cs b/KubernetesWorkflow/CrashWatcher.cs index 86f8ae1..7f8bc71 100644 --- a/KubernetesWorkflow/CrashWatcher.cs +++ b/KubernetesWorkflow/CrashWatcher.cs @@ -43,6 +43,12 @@ namespace KubernetesWorkflow if (workerException != null) throw new Exception("Exception occurred in CrashWatcher worker thread.", workerException); } + public bool HasContainerCrashed() + { + using var client = new Kubernetes(config); + return HasContainerBeenRestarted(client, container.Pod.PodInfo.Name); + } + private void Worker() { try @@ -57,7 +63,7 @@ namespace KubernetesWorkflow private void MonitorContainer(CancellationToken token) { - var client = new Kubernetes(config); + using var client = new Kubernetes(config); while (!token.IsCancellationRequested) { token.WaitHandle.WaitOne(TimeSpan.FromSeconds(1)); @@ -65,9 +71,7 @@ namespace KubernetesWorkflow var pod = container.Pod; var recipe = container.Recipe; var podName = pod.PodInfo.Name; - var podInfo = client.ReadNamespacedPod(podName, k8sNamespace); - - if (podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0)) + if (HasContainerBeenRestarted(client, podName)) { DownloadCrashedContainerLogs(client, podName, recipe); return; @@ -75,6 +79,12 @@ namespace KubernetesWorkflow } } + private bool HasContainerBeenRestarted(Kubernetes client, string podName) + { + var podInfo = client.ReadNamespacedPod(podName, k8sNamespace); + return podInfo.Status.ContainerStatuses.Any(c => c.RestartCount > 0); + } + private void DownloadCrashedContainerLogs(Kubernetes client, string podName, ContainerRecipe recipe) { log.Log("Pod crash detected for " + container.Name); From 3aea8f41b812d6329494ef39a571e8c807f83c57 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 29 Aug 2023 10:09:48 +0200 Subject: [PATCH 8/9] wires up timing measurement for peer download helper. --- DistTestCore/Codex/CodexAccess.cs | 5 ++--- DistTestCore/Helpers/FullConnectivityHelper.cs | 2 +- DistTestCore/Helpers/PeerDownloadTestHelpers.cs | 6 ++++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 57f7689..cf6ac19 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -95,11 +95,10 @@ namespace DistTestCore.Codex private Http Http() { - CheckContainerCrashed(); - return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", Container.Name); + return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name); } - private void CheckContainerCrashed() + private void CheckContainerCrashed(HttpClient client) { if (hasContainerCrashed) throw new Exception("Container has crashed."); } diff --git a/DistTestCore/Helpers/FullConnectivityHelper.cs b/DistTestCore/Helpers/FullConnectivityHelper.cs index 228beeb..68dfdbb 100644 --- a/DistTestCore/Helpers/FullConnectivityHelper.cs +++ b/DistTestCore/Helpers/FullConnectivityHelper.cs @@ -56,7 +56,7 @@ namespace DistTestCore.Helpers private static void RetryWhilePairs(List pairs, Action action) { - var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(5); + var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(2); while (pairs.Any(p => p.Inconclusive) && timeout > DateTime.UtcNow) { action(); diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs index 40024e1..cd12e2b 100644 --- a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -7,6 +7,7 @@ namespace DistTestCore.Helpers public class PeerDownloadTestHelpers : IFullConnectivityImplementation { private readonly FullConnectivityHelper helper; + private readonly BaseLog log; private readonly FileManager fileManager; private ByteSize testFileSize; @@ -14,6 +15,7 @@ namespace DistTestCore.Helpers { helper = new FullConnectivityHelper(log, this); testFileSize = 1.MB(); + this.log = log; this.fileManager = fileManager; } @@ -44,11 +46,11 @@ namespace DistTestCore.Helpers var expectedFile = GenerateTestFile(from.Node, to.Node); using var uploadStream = File.OpenRead(expectedFile.Filename); - var contentId = from.Node.UploadFile(uploadStream); + var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(uploadStream)); try { - var downloadedFile = DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded"); + var downloadedFile = Stopwatch.Measure(log, "Download", () => DownloadFile(to.Node, contentId, expectedFile.Label + "_downloaded")); expectedFile.AssertIsEqual(downloadedFile); return PeerConnectionState.Connection; } From facfd10316bdfd9733f2656f60f5b17ff06d0de3 Mon Sep 17 00:00:00 2001 From: benbierens Date: Tue, 29 Aug 2023 14:48:03 +0200 Subject: [PATCH 9/9] Removes log asserts, they are unreliable in the cluster. --- ContinuousTests/Tests/HoldMyBeerTest.cs | 69 ------------------------- 1 file changed, 69 deletions(-) diff --git a/ContinuousTests/Tests/HoldMyBeerTest.cs b/ContinuousTests/Tests/HoldMyBeerTest.cs index f54108e..18e72f1 100644 --- a/ContinuousTests/Tests/HoldMyBeerTest.cs +++ b/ContinuousTests/Tests/HoldMyBeerTest.cs @@ -1,5 +1,4 @@ using DistTestCore; -using Logging; using NUnit.Framework; namespace ContinuousTests.Tests @@ -26,7 +25,6 @@ namespace ContinuousTests.Tests var filesize = filesizePerUploadMb.MB(); double codexDefaultBlockSize = 31 * 64 * 33; var numberOfBlocks = Convert.ToInt64(Math.Ceiling(filesize.SizeInBytes / codexDefaultBlockSize)); - var sizeInBytes = filesize.SizeInBytes; Assert.That(numberOfBlocks, Is.EqualTo(1282)); file = FileManager.GenerateTestFile(filesize); @@ -34,75 +32,8 @@ namespace ContinuousTests.Tests cid = UploadFile(Nodes[0], file); Assert.That(cid, Is.Not.Null); - var cidTag = cid!.Id.Substring(cid.Id.Length - 6); - Stopwatch.Measure(Log, "upload-log-asserts", () => - { - var uploadLog = DownloadContainerLog(Nodes[0].Container, 50000); - - var storeLines = uploadLog.FindLinesThatContain("Stored data", "topics=\"codex node\""); - uploadLog.DeleteFile(); - - var storeLine = GetLineForCidTag(storeLines, cidTag); - AssertStoreLineContains(storeLine, numberOfBlocks, sizeInBytes); - }); - var dl = DownloadFile(Nodes[0], cid!); file.AssertIsEqual(dl); - - Stopwatch.Measure(Log, "download-log-asserts", () => - { - var downloadLog = DownloadContainerLog(Nodes[0].Container, 50000); - - var sentLines = downloadLog.FindLinesThatContain("Sent bytes", "topics=\"codex restapi\""); - downloadLog.DeleteFile(); - - var sentLine = GetLineForCidTag(sentLines, cidTag); - AssertSentLineContains(sentLine, sizeInBytes); - }); - } - - private void AssertSentLineContains(string sentLine, long sizeInBytes) - { - var tag = "bytes="; - var token = sentLine.Substring(sentLine.IndexOf(tag) + tag.Length); - var bytes = Convert.ToInt64(token); - Assert.AreEqual(sizeInBytes, bytes, $"Sent bytes: Number of bytes incorrect. Line: '{sentLine}'"); - } - - private void AssertStoreLineContains(string storeLine, long numberOfBlocks, long sizeInBytes) - { - var tokens = storeLine.Split(" "); - - var blocksToken = GetToken(tokens, "blocks="); - var sizeToken = GetToken(tokens, "size="); - if (blocksToken == null) Assert.Fail("blockToken not found in " + storeLine); - if (sizeToken == null) Assert.Fail("sizeToken not found in " + storeLine); - - var blocks = Convert.ToInt64(blocksToken); - var size = Convert.ToInt64(sizeToken?.Replace("'NByte", "")); - - var lineLog = $" Line: '{storeLine}'"; - Assert.AreEqual(numberOfBlocks, blocks, "Stored data: Number of blocks incorrect." + lineLog); - Assert.AreEqual(sizeInBytes, size, "Stored data: Number of blocks incorrect." + lineLog); - } - - private string GetLineForCidTag(string[] lines, string cidTag) - { - var result = lines.SingleOrDefault(l => l.Contains(cidTag)); - if (result == null) - { - Assert.Fail($"Failed to find '{cidTag}' in lines: '{string.Join(",", lines)}'"); - throw new Exception(); - } - - return result; - } - - private string? GetToken(string[] tokens, string tag) - { - var token = tokens.SingleOrDefault(t => t.StartsWith(tag)); - if (token == null) return null; - return token.Substring(tag.Length); } } }