Sets up data tests

Tests/CodexReleaseTests/CodexReleaseTests.csproj

@@ -6,4 +6,12 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="NUnit" Version="3.13.3" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\CodexTests\CodexTests.csproj" />
+  </ItemGroup>
+
 </Project>

Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs

@@ -1,26 +1,16 @@
 using CodexPlugin;
+using CodexTests;
 using FileUtils;
 using NUnit.Framework;
 using System.Diagnostics;
 using Utils;
 
-namespace CodexTests.BasicTests
+namespace CodexReleaseTests.DataTests
 {
-    [TestFixture]
+    public class InterruptUploadTest : CodexDistTest
-    public class OneClientTests : CodexDistTest
     {
         [Test]
-        public void OneClientTest()
+        public void UploadInterruptTest()
-        {
-            var node = StartCodex();
-
-            PerformOneClientTest(node);
-
-            LogNodeStatus(node);
-        }
-
-        [Test]
-        public void InterruptUploadTest()
         {
             var nodes = StartCodex(10);
 
@@ -28,6 +18,8 @@ namespace CodexTests.BasicTests
             Task.WaitAll(tasks.ToArray());
 
             Assert.That(tasks.Select(t => t.Result).All(r => r == true));
+
+            WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), nodes);
         }
 
         private bool RunInterruptUploadTest(ICodexNode node)
@@ -51,16 +43,5 @@ namespace CodexTests.BasicTests
             var filePath = file.Filename;
             return Process.Start("curl", $"-X POST {codexUrl} -H \"Content-Type: application/octet-stream\" -T {filePath}");
         }
-
-        private void PerformOneClientTest(ICodexNode primary)
-        {
-            var testFile = GenerateTestFile(1.MB());
-
-            var contentId = primary.UploadFile(testFile);
-
-            var downloadedFile = primary.DownloadContent(contentId);
-
-            testFile.AssertIsEqual(downloadedFile);
-        }
     }
 }

Tests/CodexReleaseTests/DataTests/OneClientTest.cs

@@ -0,0 +1,39 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Utils;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class OneClientTest : CodexDistTest
+    {
+        [Test]
+        public void OneClient()
+        {
+            var node = StartCodex();
+
+            PerformOneClientTest(node);
+
+            LogNodeStatus(node);
+        }
+
+        private void PerformOneClientTest(ICodexNode primary)
+        {
+            var testFile = GenerateTestFile(1.MB());
+
+            var contentId = primary.UploadFile(testFile);
+
+            AssertNodesContainFile(contentId, primary);
+
+            var downloadedFile = primary.DownloadContent(contentId);
+
+            testFile.AssertIsEqual(downloadedFile);
+        }
+    }
+}

Tests/CodexReleaseTests/DataTests/SwarmTest.cs

@@ -1,12 +1,92 @@
-using System;
+using CodexPlugin;
+using CodexTests;
+using FileUtils;
+using NUnit.Framework;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Utils;
 
 namespace CodexReleaseTests.DataTests
 {
-    internal class SwarmTest
+    [TestFixture]
+    public class SwarmTests : AutoBootstrapDistTest
     {
+        private const int NumberOfNodes = 10;
+        private const int FileSizeMb = 100;
+
+        [Test]
+        public void SmallSwarm()
+        {
+            var nodes = StartCodex(NumberOfNodes);
+            var files = nodes.Select(UploadUniqueFilePerNode).ToArray();
+
+            var tasks = ParallelDownloadEachFile(nodes, files);
+            Task.WaitAll(tasks);
+
+            AssertAllFilesDownloadedCorrectly(files);
+        }
+
+        private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node)
+        {
+            var file = GenerateTestFile(FileSizeMb.MB());
+            var cid = node.UploadFile(file);
+            return new SwarmTestNetworkFile(file, cid);
+        }
+
+        private Task[] ParallelDownloadEachFile(ICodexNodeGroup nodes, SwarmTestNetworkFile[] files)
+        {
+            var tasks = new List<Task>();
+
+            foreach (var node in nodes)
+            {
+                foreach (var file in files)
+                {
+                    tasks.Add(StartDownload(node, file));
+                }
+            }
+
+            return tasks.ToArray();
+        }
+
+        private Task StartDownload(ICodexNode node, SwarmTestNetworkFile file)
+        {
+            return Task.Run(() =>
+            {
+                try
+                {
+                    file.Downloaded = node.DownloadContent(file.Cid);
+                }
+                catch (Exception ex)
+                {
+                    file.Error = ex;
+                }
+            });
+        }
+
+        private void AssertAllFilesDownloadedCorrectly(SwarmTestNetworkFile[] files)
+        {
+            foreach (var file in files)
+            {
+                if (file.Error != null) throw file.Error;
+                file.Original.AssertIsEqual(file.Downloaded);
+            }
+        }
+
+        private class SwarmTestNetworkFile
+        {
+            public SwarmTestNetworkFile(TrackedFile original, ContentId cid)
+            {
+                Original = original;
+                Cid = cid;
+            }
+
+            public TrackedFile Original { get; }
+            public ContentId Cid { get; }
+            public TrackedFile? Downloaded { get; set; }
+            public Exception? Error { get; set; } = null;
+        }
     }
 }

Tests/CodexReleaseTests/DataTests/ThreeClientTest.cs

@@ -1,12 +1,32 @@
-using System;
+using CodexTests;
+using NUnit.Framework;
+using System;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Utils;
 
 namespace CodexReleaseTests.DataTests
 {
-    internal class ThreeClientTest
+    public class ThreeClientTest : AutoBootstrapDistTest
     {
+        [Test]
+        public void ThreeClient()
+        {
+            var primary = StartCodex();
+            var secondary = StartCodex();
+
+            var testFile = GenerateTestFile(10.MB());
+
+            var contentId = primary.UploadFile(testFile);
+            AssertNodesContainFile(contentId, primary);
+
+            var downloadedFile = secondary.DownloadContent(contentId);
+            AssertNodesContainFile(contentId, primary, secondary);
+
+            testFile.AssertIsEqual(downloadedFile);
+        }
     }
 }

Tests/CodexReleaseTests/DataTests/TwoClientTest.cs

@@ -1,12 +1,60 @@
-using System;
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Utils;
 
 namespace CodexReleaseTests.DataTests
 {
-    internal class TwoClientTest
+    [TestFixture]
+    public class TwoClientTests : CodexDistTest
     {
+        [Test]
+        public void TwoClientTest()
+        {
+            var uploader = StartCodex(s => s.WithName("Uploader"));
+            var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
+
+            PerformTwoClientTest(uploader, downloader);
+        }
+
+        [Test]
+        public void TwoClientsTwoLocationsTest()
+        {
+            var locations = Ci.GetKnownLocations();
+            if (locations.NumberOfLocations < 2)
+            {
+                Assert.Inconclusive("Two-locations test requires 2 nodes to be available in the cluster.");
+                return;
+            }
+
+            var uploader = Ci.StartCodexNode(s => s.WithName("Uploader").At(locations.Get(0)));
+            var downloader = Ci.StartCodexNode(s => s.WithName("Downloader").WithBootstrapNode(uploader).At(locations.Get(1)));
+
+            PerformTwoClientTest(uploader, downloader);
+        }
+
+        private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader)
+        {
+            PerformTwoClientTest(uploader, downloader, 10.MB());
+        }
+
+        private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader, ByteSize size)
+        {
+            var testFile = GenerateTestFile(size);
+
+            var contentId = uploader.UploadFile(testFile);
+            AssertNodesContainFile(contentId, uploader);
+
+            var downloadedFile = downloader.DownloadContent(contentId);
+            AssertNodesContainFile(contentId, uploader, downloader);
+
+            testFile.AssertIsEqual(downloadedFile);
+            CheckLogForErrors(uploader, downloader);
+        }
     }
 }

Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs

@@ -0,0 +1,40 @@
+using CodexPlugin;
+using CodexTests;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CodexReleaseTests.DataTests
+{
+    [TestFixture]
+    public class UnknownCidTest : CodexDistTest
+    {
+        [Test]
+        public void DownloadingUnknownCidDoesNotCauseCrash()
+        {
+            var node = StartCodex();
+
+            var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ");
+
+            var localFiles = node.LocalFiles().Content;
+            CollectionAssert.DoesNotContain(localFiles.Select(f => f.Cid), unknownCid);
+
+            try
+            {
+                node.DownloadContent(unknownCid);
+            }
+            catch (Exception ex)
+            {
+                if (!ex.Message.StartsWith("Retry 'DownloadFile' timed out"))
+                {
+                    throw;
+                }
+            }
+
+            WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), node);
+        }
+    }
+}

Tests/CodexTests/BasicTests/ThreeClientTest.cs

@@ -1,54 +0,0 @@
-using CodexPlugin;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.BasicTests
-{
-    [TestFixture]
-    public class ThreeClientTest : AutoBootstrapDistTest
-    {
-        [Test]
-        public void ThreeClient()
-        {
-            var primary = StartCodex();
-            var secondary = StartCodex();
-
-            var testFile = GenerateTestFile(10.MB());
-
-            var contentId = primary.UploadFile(testFile);
-
-            var downloadedFile = secondary.DownloadContent(contentId);
-
-            testFile.AssertIsEqual(downloadedFile);
-        }
-
-        [Test]
-        public void DownloadingUnknownCidDoesNotCauseCrash()
-        {
-            var node = StartCodex(2).First();
-
-            var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ");
-
-            try
-            {
-                node.DownloadContent(unknownCid);
-            }
-            catch (Exception ex)
-            {
-                if (!ex.Message.StartsWith("Retry 'DownloadFile' timed out"))
-                {
-                    throw;
-                }
-            }
-
-            // Check that the node stays alive for at least another 5 minutes.
-            var start = DateTime.UtcNow;
-            while ((DateTime.UtcNow - start) < TimeSpan.FromMinutes(5))
-            {
-                Thread.Sleep(5000);
-                var info = node.GetDebugInfo();
-                Assert.That(!string.IsNullOrEmpty(info.Id));
-            }
-        }
-    }
-}

Tests/CodexTests/BasicTests/TwoClientTests.cs

@@ -1,52 +0,0 @@
-using CodexPlugin;
-using NUnit.Framework;
-using Utils;
-
-namespace CodexTests.BasicTests
-{
-    [TestFixture]
-    public class TwoClientTests : CodexDistTest
-    {
-        [Test]
-        public void TwoClientTest()
-        {
-            var uploader = StartCodex(s => s.WithName("Uploader"));
-            var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader));
-
-            PerformTwoClientTest(uploader, downloader);
-        }
-
-        [Test]
-        public void TwoClientsTwoLocationsTest()
-        {
-            var locations = Ci.GetKnownLocations();
-            if (locations.NumberOfLocations < 2)
-            {
-                Assert.Inconclusive("Two-locations test requires 2 nodes to be available in the cluster.");
-                return;
-            }
-
-            var uploader = Ci.StartCodexNode(s => s.WithName("Uploader").At(locations.Get(0)));
-            var downloader = Ci.StartCodexNode(s => s.WithName("Downloader").WithBootstrapNode(uploader).At(locations.Get(1)));
-
-            PerformTwoClientTest(uploader, downloader);
-        }
-
-        private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader)
-        {
-            PerformTwoClientTest(uploader, downloader, 10.MB());
-        }
-
-        private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader, ByteSize size)
-        {
-            var testFile = GenerateTestFile(size);
-
-            var contentId = uploader.UploadFile(testFile);
-
-            var downloadedFile = downloader.DownloadContent(contentId);
-
-            testFile.AssertIsEqual(downloadedFile);
-            CheckLogForErrors(uploader, downloader);
-        }
-    }
-}

Tests/CodexTests/CodexDistTest.cs

@@ -108,6 +108,39 @@ namespace CodexTests
                 GetBasicNodeStatus(node));
         }
 
+        public void WaitAndCheckNodesStaysAlive(TimeSpan duration, ICodexNodeGroup nodes)
+        {
+            WaitAndCheckNodesStaysAlive(duration, nodes.ToArray());
+        }
+
+        public void WaitAndCheckNodesStaysAlive(TimeSpan duration, params ICodexNode[] nodes)
+        {
+            var start = DateTime.UtcNow;
+            while ((DateTime.UtcNow - start) < duration)
+            {
+                Thread.Sleep(5000);
+                foreach (var node in nodes)
+                {
+                    var info = node.GetDebugInfo();
+                    Assert.That(!string.IsNullOrEmpty(info.Id));
+                }
+            }
+        }
+
+        public void AssertNodesContainFile(ContentId cid, ICodexNodeGroup nodes)
+        {
+            AssertNodesContainFile(cid, nodes.ToArray());
+        }
+
+        public void AssertNodesContainFile(ContentId cid, params ICodexNode[] nodes)
+        {
+            foreach (var node in nodes)
+            {
+                var localDatasets = node.LocalFiles();
+                CollectionAssert.Contains(localDatasets.Content.Select(c => c.Cid), cid);
+            }
+        }
+
         private string GetBasicNodeStatus(ICodexNode node)
         {
             return JsonConvert.SerializeObject(node.GetDebugInfo(), Formatting.Indented) + Environment.NewLine +

Tests/CodexTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs

@@ -4,7 +4,7 @@ using Utils;
 namespace CodexTests.DownloadConnectivityTests
 {
     [TestFixture]
-    public class SwarmTests : AutoBootstrapDistTest
+    public class DetectBlockRetransmitTest : AutoBootstrapDistTest
     {
         [Test]
         [Combinatorial]