From 10a0ac33117f2aa4e4fb65f296e89aff65b4963d Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 4 Jun 2023 08:59:51 +0200 Subject: [PATCH] Splits up download tests from connectivity tests and improves testfile logging --- DistTestCore/Codex/CodexContainerRecipe.cs | 10 +++--- DistTestCore/DistTest.cs | 4 +-- DistTestCore/FileManager.cs | 34 ++++++++++++++----- .../Helpers/PeerDownloadTestHelpers.cs | 14 ++++++-- DistTestCore/OnlineCodexNode.cs | 11 +++--- .../FullyConnectedDownloadTests.cs | 19 +++++++++++ .../LayeredDiscoveryTests.cs | 1 - .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 1 - 8 files changed, 68 insertions(+), 26 deletions(-) create mode 100644 Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index e92c93c..5b25a2f 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -5,12 +5,12 @@ namespace DistTestCore.Codex { public class CodexContainerRecipe : ContainerRecipeFactory { - #if Arm64 +#if Arm64 public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0"; - #else - //public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0"; - public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0"; - #endif +#else + public const string DockerImage = "thatbenbierens/nim-codex:dhting"; + //public const string DockerImage = "codexstorage/nim-codex:sha-7b88ea0"; +#endif public const string MetricsPortTag = "metrics_port"; public const string DiscoveryPortTag = "discovery-port"; diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index bf4d59f..df345e8 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -87,9 +87,9 @@ namespace DistTestCore } } - public TestFile GenerateTestFile(ByteSize size) + public TestFile GenerateTestFile(ByteSize size, string label = "") { - return Get().FileManager.GenerateTestFile(size); + return Get().FileManager.GenerateTestFile(size, label); } /// diff --git a/DistTestCore/FileManager.cs b/DistTestCore/FileManager.cs index 97cf27c..62b5ba7 100644 --- a/DistTestCore/FileManager.cs +++ b/DistTestCore/FileManager.cs @@ -6,8 +6,8 @@ namespace DistTestCore { public interface IFileManager { - TestFile CreateEmptyTestFile(); - TestFile GenerateTestFile(ByteSize size); + TestFile CreateEmptyTestFile(string label = ""); + TestFile GenerateTestFile(ByteSize size, string label = ""); void DeleteAllTestFiles(); void PushFileSet(); void PopFileSet(); @@ -30,19 +30,20 @@ namespace DistTestCore this.log = log; } - public TestFile CreateEmptyTestFile() + public TestFile CreateEmptyTestFile(string label = "") { - var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin")); + var path = Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin"); + var result = new TestFile(log, path, label); File.Create(result.Filename).Close(); if (fileSetStack.Any()) fileSetStack.Last().Add(result); return result; } - public TestFile GenerateTestFile(ByteSize size) + public TestFile GenerateTestFile(ByteSize size, string label) { - var result = CreateEmptyTestFile(); + var result = CreateEmptyTestFile(label); GenerateFileBytes(result, size); - log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Filename}'."); + log.Log($"Generated {size.SizeInBytes} bytes of content for file '{result.Describe()}'."); return result; } @@ -104,12 +105,17 @@ namespace DistTestCore public class TestFile { - public TestFile(string filename) + private readonly TestLog log; + + public TestFile(TestLog log, string filename, string label) { + this.log = log; Filename = filename; + Label = label; } public string Filename { get; } + public string Label { get; } public long GetFileSize() { @@ -138,10 +144,20 @@ namespace DistTestCore readExpected = streamExpected.Read(bytesExpected, 0, FileManager.ChunkSize); readActual = streamActual.Read(bytesActual, 0, FileManager.ChunkSize); - if (readExpected == 0 && readActual == 0) return; + if (readExpected == 0 && readActual == 0) + { + log.Log($"OK: '{Describe()}' is equal to '{actual.Describe()}'."); + return; + } Assert.That(readActual, Is.EqualTo(readExpected), "Unable to read buffers of equal length."); CollectionAssert.AreEqual(bytesExpected, bytesActual, "Files are not binary-equal."); } } + + public string Describe() + { + if (!string.IsNullOrEmpty(Label)) return Label; + return Filename; + } } } diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs index 6929ceb..c218ac0 100644 --- a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -33,8 +33,8 @@ private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize) { - // 1 test file per downloader. - var files = downloaders.Select(d => test.GenerateTestFile(testFileSize)).ToArray(); + // Generate 1 test file per downloader. + var files = downloaders.Select(d => GenerateTestFile(uploader, d, testFileSize)).ToArray(); // Upload all the test files to the uploader. var contentIds = files.Select(uploader.UploadFile).ToArray(); @@ -43,10 +43,18 @@ for (var i = 0; i < downloaders.Length; i++) { var expectedFile = files[i]; - var downloadedFile = downloaders[i].DownloadContent(contentIds[i]); + var downloadedFile = downloaders[i].DownloadContent(contentIds[i], $"{expectedFile.Label}DOWNLOADED"); expectedFile.AssertIsEqual(downloadedFile); } } + + private TestFile GenerateTestFile(IOnlineCodexNode uploader, IOnlineCodexNode downloader, ByteSize testFileSize) + { + var up = uploader.GetName().Replace("<", "").Replace(">", ""); + var down = downloader.GetName().Replace("<", "").Replace(">", ""); + var label = $"FROM{up}TO{down}"; + return test.GenerateTestFile(testFileSize, label); + } } } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 6e12c79..a1197f6 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -13,7 +13,7 @@ namespace DistTestCore CodexDebugPeerResponse GetDebugPeer(string peerId); CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout); ContentId UploadFile(TestFile file); - TestFile? DownloadContent(ContentId contentId); + TestFile? DownloadContent(ContentId contentId, string fileLabel = ""); void ConnectToPeer(IOnlineCodexNode node); ICodexNodeLog DownloadLog(); IMetricsAccess Metrics { get; } @@ -66,23 +66,24 @@ namespace DistTestCore public ContentId UploadFile(TestFile file) { - Log($"Uploading file of size {file.GetFileSize()}..."); + Log($"Uploading file '{file.Describe()}' of size {file.GetFileSize()}..."); using var fileStream = File.OpenRead(file.Filename); var response = CodexAccess.UploadFile(fileStream); if (response.StartsWith(UploadFailedMessage)) { Assert.Fail("Node failed to store block."); } + lifecycle.Log.AddStringReplace(response, $"(CID:{file.Describe()})"); Log($"Uploaded file. Received contentId: '{response}'."); return new ContentId(response); } - public TestFile? DownloadContent(ContentId contentId) + public TestFile? DownloadContent(ContentId contentId, string fileLabel = "") { Log($"Downloading for contentId: '{contentId.Id}'..."); - var file = lifecycle.FileManager.CreateEmptyTestFile(); + var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel); DownloadToFile(contentId.Id, file); - Log($"Downloaded file of size {file.GetFileSize()} to '{file.Filename}'."); + Log($"Downloaded file '{file.Describe()}' of size {file.GetFileSize()} to '{file.Filename}'."); return file; } diff --git a/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs new file mode 100644 index 0000000..b1e9bd6 --- /dev/null +++ b/Tests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -0,0 +1,19 @@ +using DistTestCore; +using NUnit.Framework; + +namespace Tests.DownloadConnectivityTests +{ + [TestFixture] + public class FullyConnectedDownloadTests : AutoBootstrapDistTest + { + [TestCase(3)] + [TestCase(10)] + [TestCase(20)] + public void FullyConnectedDownloadTest(int numberOfNodes) + { + for (var i = 0; i < numberOfNodes; i++) SetupCodexNode(); + + PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes()); + } + } +} diff --git a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index 306d305..ad04144 100644 --- a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -56,7 +56,6 @@ namespace Tests.PeerDiscoveryTests private void AssertAllNodesConnected() { PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); - //PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes()); } } } diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs index a77cc3a..841b072 100644 --- a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -67,7 +67,6 @@ namespace Tests.PeerDiscoveryTests private void AssertAllNodesConnected() { PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); - //PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes()); } } }