diff --git a/Tests/CodexReleaseTests/CodexReleaseTests.csproj b/Tests/CodexReleaseTests/CodexReleaseTests.csproj index fa71b7ae..269cf26f 100644 --- a/Tests/CodexReleaseTests/CodexReleaseTests.csproj +++ b/Tests/CodexReleaseTests/CodexReleaseTests.csproj @@ -6,4 +6,12 @@ enable + + + + + + + + diff --git a/Tests/CodexTests/BasicTests/OneClientTests.cs b/Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs similarity index 66% rename from Tests/CodexTests/BasicTests/OneClientTests.cs rename to Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs index 27c855ab..ae9c9d58 100644 --- a/Tests/CodexTests/BasicTests/OneClientTests.cs +++ b/Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs @@ -1,26 +1,16 @@ using CodexPlugin; +using CodexTests; using FileUtils; using NUnit.Framework; using System.Diagnostics; using Utils; -namespace CodexTests.BasicTests +namespace CodexReleaseTests.DataTests { - [TestFixture] - public class OneClientTests : CodexDistTest + public class InterruptUploadTest : CodexDistTest { [Test] - public void OneClientTest() - { - var node = StartCodex(); - - PerformOneClientTest(node); - - LogNodeStatus(node); - } - - [Test] - public void InterruptUploadTest() + public void UploadInterruptTest() { var nodes = StartCodex(10); @@ -28,6 +18,8 @@ namespace CodexTests.BasicTests Task.WaitAll(tasks.ToArray()); Assert.That(tasks.Select(t => t.Result).All(r => r == true)); + + WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), nodes); } private bool RunInterruptUploadTest(ICodexNode node) @@ -51,16 +43,5 @@ namespace CodexTests.BasicTests var filePath = file.Filename; return Process.Start("curl", $"-X POST {codexUrl} -H \"Content-Type: application/octet-stream\" -T {filePath}"); } - - private void PerformOneClientTest(ICodexNode primary) - { - var testFile = GenerateTestFile(1.MB()); - - var contentId = primary.UploadFile(testFile); - - var downloadedFile = primary.DownloadContent(contentId); - - testFile.AssertIsEqual(downloadedFile); - } } } diff --git a/Tests/CodexReleaseTests/DataTests/OneClientTest.cs b/Tests/CodexReleaseTests/DataTests/OneClientTest.cs new file mode 100644 index 00000000..1be82b4d --- /dev/null +++ b/Tests/CodexReleaseTests/DataTests/OneClientTest.cs @@ -0,0 +1,39 @@ +using CodexPlugin; +using CodexTests; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Utils; + +namespace CodexReleaseTests.DataTests +{ + [TestFixture] + public class OneClientTest : CodexDistTest + { + [Test] + public void OneClient() + { + var node = StartCodex(); + + PerformOneClientTest(node); + + LogNodeStatus(node); + } + + private void PerformOneClientTest(ICodexNode primary) + { + var testFile = GenerateTestFile(1.MB()); + + var contentId = primary.UploadFile(testFile); + + AssertNodesContainFile(contentId, primary); + + var downloadedFile = primary.DownloadContent(contentId); + + testFile.AssertIsEqual(downloadedFile); + } + } +} diff --git a/Tests/CodexReleaseTests/DataTests/SwarmTest.cs b/Tests/CodexReleaseTests/DataTests/SwarmTest.cs index 9df80363..a0e9be90 100644 --- a/Tests/CodexReleaseTests/DataTests/SwarmTest.cs +++ b/Tests/CodexReleaseTests/DataTests/SwarmTest.cs @@ -1,12 +1,92 @@ -using System; +using CodexPlugin; +using CodexTests; +using FileUtils; +using NUnit.Framework; +using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; +using Utils; namespace CodexReleaseTests.DataTests { - internal class SwarmTest + [TestFixture] + public class SwarmTests : AutoBootstrapDistTest { + private const int NumberOfNodes = 10; + private const int FileSizeMb = 100; + + [Test] + public void SmallSwarm() + { + var nodes = StartCodex(NumberOfNodes); + var files = nodes.Select(UploadUniqueFilePerNode).ToArray(); + + var tasks = ParallelDownloadEachFile(nodes, files); + Task.WaitAll(tasks); + + AssertAllFilesDownloadedCorrectly(files); + } + + private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node) + { + var file = GenerateTestFile(FileSizeMb.MB()); + var cid = node.UploadFile(file); + return new SwarmTestNetworkFile(file, cid); + } + + private Task[] ParallelDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files) + { + var tasks = new List(); + + foreach (var node in nodes) + { + foreach (var file in files) + { + tasks.Add(StartDownload(node, file)); + } + } + + return tasks.ToArray(); + } + + private Task StartDownload(ICodexNode node, SwarmTestNetworkFile file) + { + return Task.Run(() => + { + try + { + file.Downloaded = node.DownloadContent(file.Cid); + } + catch (Exception ex) + { + file.Error = ex; + } + }); + } + + private void AssertAllFilesDownloadedCorrectly(SwarmTestNetworkFile[] files) + { + foreach (var file in files) + { + if (file.Error != null) throw file.Error; + file.Original.AssertIsEqual(file.Downloaded); + } + } + + private class SwarmTestNetworkFile + { + public SwarmTestNetworkFile(TrackedFile original, ContentId cid) + { + Original = original; + Cid = cid; + } + + public TrackedFile Original { get; } + public ContentId Cid { get; } + public TrackedFile? Downloaded { get; set; } + public Exception? Error { get; set; } = null; + } } } diff --git a/Tests/CodexReleaseTests/DataTests/ThreeClientTest.cs b/Tests/CodexReleaseTests/DataTests/ThreeClientTest.cs index 64e2cc4c..a99f0ae1 100644 --- a/Tests/CodexReleaseTests/DataTests/ThreeClientTest.cs +++ b/Tests/CodexReleaseTests/DataTests/ThreeClientTest.cs @@ -1,12 +1,32 @@ -using System; +using CodexTests; +using NUnit.Framework; +using System; using System.Collections.Generic; +using System.ComponentModel; using System.Linq; using System.Text; using System.Threading.Tasks; +using Utils; namespace CodexReleaseTests.DataTests { - internal class ThreeClientTest + public class ThreeClientTest : AutoBootstrapDistTest { + [Test] + public void ThreeClient() + { + var primary = StartCodex(); + var secondary = StartCodex(); + + var testFile = GenerateTestFile(10.MB()); + + var contentId = primary.UploadFile(testFile); + AssertNodesContainFile(contentId, primary); + + var downloadedFile = secondary.DownloadContent(contentId); + AssertNodesContainFile(contentId, primary, secondary); + + testFile.AssertIsEqual(downloadedFile); + } } } diff --git a/Tests/CodexReleaseTests/DataTests/TwoClientTest.cs b/Tests/CodexReleaseTests/DataTests/TwoClientTest.cs index 578bb5fd..c2bf324f 100644 --- a/Tests/CodexReleaseTests/DataTests/TwoClientTest.cs +++ b/Tests/CodexReleaseTests/DataTests/TwoClientTest.cs @@ -1,12 +1,60 @@ -using System; +using CodexPlugin; +using CodexTests; +using NUnit.Framework; +using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; +using Utils; namespace CodexReleaseTests.DataTests { - internal class TwoClientTest + [TestFixture] + public class TwoClientTests : CodexDistTest { + [Test] + public void TwoClientTest() + { + var uploader = StartCodex(s => s.WithName("Uploader")); + var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader)); + + PerformTwoClientTest(uploader, downloader); + } + + [Test] + public void TwoClientsTwoLocationsTest() + { + var locations = Ci.GetKnownLocations(); + if (locations.NumberOfLocations < 2) + { + Assert.Inconclusive("Two-locations test requires 2 nodes to be available in the cluster."); + return; + } + + var uploader = Ci.StartCodexNode(s => s.WithName("Uploader").At(locations.Get(0))); + var downloader = Ci.StartCodexNode(s => s.WithName("Downloader").WithBootstrapNode(uploader).At(locations.Get(1))); + + PerformTwoClientTest(uploader, downloader); + } + + private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader) + { + PerformTwoClientTest(uploader, downloader, 10.MB()); + } + + private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader, ByteSize size) + { + var testFile = GenerateTestFile(size); + + var contentId = uploader.UploadFile(testFile); + AssertNodesContainFile(contentId, uploader); + + var downloadedFile = downloader.DownloadContent(contentId); + AssertNodesContainFile(contentId, uploader, downloader); + + testFile.AssertIsEqual(downloadedFile); + CheckLogForErrors(uploader, downloader); + } } } diff --git a/Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs b/Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs new file mode 100644 index 00000000..a7f0e66a --- /dev/null +++ b/Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs @@ -0,0 +1,40 @@ +using CodexPlugin; +using CodexTests; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CodexReleaseTests.DataTests +{ + [TestFixture] + public class UnknownCidTest : CodexDistTest + { + [Test] + public void DownloadingUnknownCidDoesNotCauseCrash() + { + var node = StartCodex(); + + var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ"); + + var localFiles = node.LocalFiles().Content; + CollectionAssert.DoesNotContain(localFiles.Select(f => f.Cid), unknownCid); + + try + { + node.DownloadContent(unknownCid); + } + catch (Exception ex) + { + if (!ex.Message.StartsWith("Retry 'DownloadFile' timed out")) + { + throw; + } + } + + WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), node); + } + } +} diff --git a/Tests/CodexTests/BasicTests/ThreeClientTest.cs b/Tests/CodexTests/BasicTests/ThreeClientTest.cs deleted file mode 100644 index e6f9428e..00000000 --- a/Tests/CodexTests/BasicTests/ThreeClientTest.cs +++ /dev/null @@ -1,54 +0,0 @@ -using CodexPlugin; -using NUnit.Framework; -using Utils; - -namespace CodexTests.BasicTests -{ - [TestFixture] - public class ThreeClientTest : AutoBootstrapDistTest - { - [Test] - public void ThreeClient() - { - var primary = StartCodex(); - var secondary = StartCodex(); - - var testFile = GenerateTestFile(10.MB()); - - var contentId = primary.UploadFile(testFile); - - var downloadedFile = secondary.DownloadContent(contentId); - - testFile.AssertIsEqual(downloadedFile); - } - - [Test] - public void DownloadingUnknownCidDoesNotCauseCrash() - { - var node = StartCodex(2).First(); - - var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ"); - - try - { - node.DownloadContent(unknownCid); - } - catch (Exception ex) - { - if (!ex.Message.StartsWith("Retry 'DownloadFile' timed out")) - { - throw; - } - } - - // Check that the node stays alive for at least another 5 minutes. - var start = DateTime.UtcNow; - while ((DateTime.UtcNow - start) < TimeSpan.FromMinutes(5)) - { - Thread.Sleep(5000); - var info = node.GetDebugInfo(); - Assert.That(!string.IsNullOrEmpty(info.Id)); - } - } - } -} diff --git a/Tests/CodexTests/BasicTests/TwoClientTests.cs b/Tests/CodexTests/BasicTests/TwoClientTests.cs deleted file mode 100644 index ab15ff25..00000000 --- a/Tests/CodexTests/BasicTests/TwoClientTests.cs +++ /dev/null @@ -1,52 +0,0 @@ -using CodexPlugin; -using NUnit.Framework; -using Utils; - -namespace CodexTests.BasicTests -{ - [TestFixture] - public class TwoClientTests : CodexDistTest - { - [Test] - public void TwoClientTest() - { - var uploader = StartCodex(s => s.WithName("Uploader")); - var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader)); - - PerformTwoClientTest(uploader, downloader); - } - - [Test] - public void TwoClientsTwoLocationsTest() - { - var locations = Ci.GetKnownLocations(); - if (locations.NumberOfLocations < 2) - { - Assert.Inconclusive("Two-locations test requires 2 nodes to be available in the cluster."); - return; - } - - var uploader = Ci.StartCodexNode(s => s.WithName("Uploader").At(locations.Get(0))); - var downloader = Ci.StartCodexNode(s => s.WithName("Downloader").WithBootstrapNode(uploader).At(locations.Get(1))); - - PerformTwoClientTest(uploader, downloader); - } - - private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader) - { - PerformTwoClientTest(uploader, downloader, 10.MB()); - } - - private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader, ByteSize size) - { - var testFile = GenerateTestFile(size); - - var contentId = uploader.UploadFile(testFile); - - var downloadedFile = downloader.DownloadContent(contentId); - - testFile.AssertIsEqual(downloadedFile); - CheckLogForErrors(uploader, downloader); - } - } -} diff --git a/Tests/CodexTests/CodexDistTest.cs b/Tests/CodexTests/CodexDistTest.cs index b17a37a1..d8f75e84 100644 --- a/Tests/CodexTests/CodexDistTest.cs +++ b/Tests/CodexTests/CodexDistTest.cs @@ -108,6 +108,39 @@ namespace CodexTests GetBasicNodeStatus(node)); } + public void WaitAndCheckNodesStaysAlive(TimeSpan duration, ICodexNodeGroup nodes) + { + WaitAndCheckNodesStaysAlive(duration, nodes.ToArray()); + } + + public void WaitAndCheckNodesStaysAlive(TimeSpan duration, params ICodexNode[] nodes) + { + var start = DateTime.UtcNow; + while ((DateTime.UtcNow - start) < duration) + { + Thread.Sleep(5000); + foreach (var node in nodes) + { + var info = node.GetDebugInfo(); + Assert.That(!string.IsNullOrEmpty(info.Id)); + } + } + } + + public void AssertNodesContainFile(ContentId cid, ICodexNodeGroup nodes) + { + AssertNodesContainFile(cid, nodes.ToArray()); + } + + public void AssertNodesContainFile(ContentId cid, params ICodexNode[] nodes) + { + foreach (var node in nodes) + { + var localDatasets = node.LocalFiles(); + CollectionAssert.Contains(localDatasets.Content.Select(c => c.Cid), cid); + } + } + private string GetBasicNodeStatus(ICodexNode node) { return JsonConvert.SerializeObject(node.GetDebugInfo(), Formatting.Indented) + Environment.NewLine + diff --git a/Tests/CodexTests/DownloadConnectivityTests/SwarmTests.cs b/Tests/CodexTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs similarity index 90% rename from Tests/CodexTests/DownloadConnectivityTests/SwarmTests.cs rename to Tests/CodexTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs index ee4ad46f..6f007c4e 100644 --- a/Tests/CodexTests/DownloadConnectivityTests/SwarmTests.cs +++ b/Tests/CodexTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs @@ -4,7 +4,7 @@ using Utils; namespace CodexTests.DownloadConnectivityTests { [TestFixture] - public class SwarmTests : AutoBootstrapDistTest + public class DetectBlockRetransmitTest : AutoBootstrapDistTest { [Test] [Combinatorial]