Sets up the debug/fetch handle and fully-connected test helper.

DistTestCore/Codex/CodexAccess.cs

@@ -47,6 +47,12 @@ namespace DistTestCore.Codex
             return result;
         }
 
+        public CodexDebugFetchResponse DebugFetch(string contentId)
+        {
+            var http = Http();
+            return http.HttpGetJson<CodexDebugFetchResponse>($"debug/fetch/{contentId}");
+        }
+
         public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
         {
             return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");

DistTestCore/Codex/CodexApiTypes.cs

@@ -84,6 +84,23 @@ namespace DistTestCore.Codex
         public string address { get; set; } = string.Empty;
     }
 
+    public class CodexDebugFetchResponse
+    {
+        public string originalBytes { get; set; } = string.Empty;
+        public string blockSize { get; set; } = string.Empty;
+        public string numberOfBlocks { get; set; } = string.Empty;
+        public string version { get; set; } = string.Empty;
+        public string hcodec { get; set; } = string.Empty;
+        public string codec { get; set; } = string.Empty;
+        public string @protected { get; set; } = string.Empty;
+        public CodexDebugFetchBlockResponse[] blocks { get; set; } = Array.Empty<CodexDebugFetchBlockResponse>();
+    }
+
+    public class CodexDebugFetchBlockResponse
+    {
+        public string cid { get; set; } = string.Empty;
+    }
+
     public class CodexDebugThresholdBreaches
     {
         public string[] breaches { get; set; } = Array.Empty<string>();

DistTestCore/Codex/CodexContainerRecipe.cs

@@ -5,7 +5,8 @@ namespace DistTestCore.Codex
 {
     public class CodexContainerRecipe : DefaultContainerRecipe
     {
-        private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
+        //private const string DefaultDockerImage = "codexstorage/nim-codex:sha-d279eeb-dist-tests";
+        private const string DefaultDockerImage = "thatbenbierens/nim-codex:debugfetch";
 
         public const string MetricsPortTag = "metrics_port";
         public const string DiscoveryPortTag = "discovery-port";

DistTestCore/DistTest.cs

@@ -179,6 +179,11 @@ namespace DistTestCore
             return new PeerDownloadTestHelpers(GetTestLog(), Get().FileManager);
         }
 
+        public PeerFetchTestHelpers CreatePeerFetchTestHelpers()
+        {
+            return new PeerFetchTestHelpers(GetTestLog(), Get().FileManager);
+        }
+
         public void Measure(string name, Action action)
         {
             Stopwatch.Measure(Get().Log, name, action);

DistTestCore/Helpers/PeerFetchTestHelpers.cs

@@ -0,0 +1,96 @@
+using DistTestCore.Codex;
+using Logging;
+using NUnit.Framework;
+using static DistTestCore.Helpers.FullConnectivityHelper;
+
+namespace DistTestCore.Helpers
+{
+    public class PeerFetchTestHelpers : IFullConnectivityImplementation
+    {
+        private readonly FullConnectivityHelper helper;
+        private readonly BaseLog log;
+        private readonly FileManager fileManager;
+        private readonly ByteSize testFileSize;
+        private readonly int expectedNumberOfBlocks;
+
+        public PeerFetchTestHelpers(BaseLog log, FileManager fileManager)
+        {
+            helper = new FullConnectivityHelper(log, this);
+            testFileSize = 10.MB();
+            expectedNumberOfBlocks = 161;
+            this.log = log;
+            this.fileManager = fileManager;
+        }
+
+        public void AssertFullFetchInterconnectivity(IEnumerable<IOnlineCodexNode> nodes)
+        {
+            AssertFullFetchInterconnectivity(nodes.Select(n => ((OnlineCodexNode)n).CodexAccess));
+        }
+
+        public void AssertFullFetchInterconnectivity(IEnumerable<CodexAccess> nodes)
+        {
+            helper.AssertFullyConnected(nodes);
+        }
+
+        public string Description()
+        {
+            return "Fetch connectivity";
+        }
+
+        public string ValidateEntry(Entry entry, Entry[] allEntries)
+        {
+            return string.Empty;
+        }
+
+        public PeerConnectionState Check(Entry from, Entry to)
+        {
+            fileManager.PushFileSet();
+            var expectedFile = GenerateTestFile(from.Node, to.Node);
+
+            using var uploadStream = File.OpenRead(expectedFile.Filename);
+            var contentId = Stopwatch.Measure(log, "Upload", () => from.Node.UploadFile(uploadStream));
+            var originalFetch = from.Node.DebugFetch(contentId);
+            Assert.That(Convert.ToInt32(originalFetch.numberOfBlocks), Is.EqualTo(expectedNumberOfBlocks));
+            Assert.That(originalFetch.originalBytes.Replace("'NByte", ""), Is.EqualTo("10485760"));
+
+            try
+            {
+                var fetchResponse = Stopwatch.Measure(log, "Fetch", () => to.Node.DebugFetch(contentId));
+                AssertEqual(originalFetch, fetchResponse);
+                return PeerConnectionState.Connection;
+            }
+            catch
+            {
+                return PeerConnectionState.NoConnection;
+            }
+            finally
+            {
+                fileManager.PopFileSet();
+            }
+        }
+
+        private void AssertEqual(CodexDebugFetchResponse expected, CodexDebugFetchResponse actual)
+        {
+            Assert.That(actual.originalBytes, Is.EqualTo(expected.originalBytes));
+            Assert.That(actual.blockSize, Is.EqualTo(expected.blockSize));
+            Assert.That(actual.numberOfBlocks, Is.EqualTo(expected.numberOfBlocks));
+            Assert.That(actual.version, Is.EqualTo(expected.version));
+            Assert.That(actual.hcodec, Is.EqualTo(expected.hcodec));
+            Assert.That(actual.codec, Is.EqualTo(expected.codec));
+            Assert.That(actual.@protected, Is.EqualTo(expected.@protected));
+            Assert.That(actual.blocks.Length, Is.EqualTo(expected.blocks.Length));
+            for (var i = 0; i < expected.blocks.Length; i++)
+            {
+                Assert.That(actual.blocks[i].cid, Is.EqualTo(expected.blocks[i].cid));
+            }
+        }
+
+        private TestFile GenerateTestFile(CodexAccess uploader, CodexAccess downloader)
+        {
+            var up = uploader.GetName().Replace("<", "").Replace(">", "");
+            var down = downloader.GetName().Replace("<", "").Replace(">", "");
+            var label = $"~from:{up}-to:{down}~";
+            return fileManager.GenerateTestFile(testFileSize, label);
+        }
+    }
+}

Tests/DownloadConnectivityTests/FulllyConnectedFetchTests.cs

@@ -0,0 +1,38 @@
+using DistTestCore;
+using NUnit.Framework;
+
+namespace Tests.DownloadConnectivityTests
+{
+    public class FulllyConnectedFetchTests : AutoBootstrapDistTest
+    {
+        [Test]
+        public void MetricsDoesNotInterfereWithFetch()
+        {
+            SetupCodexNodes(2, s => s.EnableMetrics());
+
+            AssertAllNodesConnected();
+        }
+
+        [Test]
+        public void MarketplaceDoesNotInterfereWithFetch()
+        {
+            SetupCodexNodes(2, s => s.EnableMetrics().EnableMarketplace(1000.TestTokens()));
+
+            AssertAllNodesConnected();
+        }
+
+        [Test]
+        [Combinatorial]
+        public void FullyConnectedFetchTest([Values(1, 3, 5)] int numberOfNodes)
+        {
+            SetupCodexNodes(numberOfNodes);
+
+            AssertAllNodesConnected();
+        }
+
+        private void AssertAllNodesConnected()
+        {
+            CreatePeerFetchTestHelpers().AssertFullFetchInterconnectivity(GetAllOnlineCodexNodes());
+        }
+    }
+}